2024-11-10 16:13:14,301 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 16:13:14,317 main DEBUG Took 0.013007 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 16:13:14,317 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 16:13:14,318 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 16:13:14,319 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 16:13:14,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,330 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 16:13:14,345 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,347 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,348 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,349 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,350 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,351 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,352 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,353 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,353 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,354 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,355 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,363 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,364 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,365 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 16:13:14,367 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,367 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 16:13:14,369 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 16:13:14,371 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 16:13:14,373 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 16:13:14,374 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 16:13:14,375 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 16:13:14,376 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 16:13:14,387 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 16:13:14,390 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 16:13:14,394 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 16:13:14,394 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 16:13:14,395 main DEBUG createAppenders(={Console}) 2024-11-10 16:13:14,399 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 16:13:14,399 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 16:13:14,400 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 16:13:14,400 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 16:13:14,401 main DEBUG OutputStream closed 2024-11-10 16:13:14,401 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 16:13:14,401 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 16:13:14,402 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 16:13:14,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 16:13:14,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 16:13:14,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 16:13:14,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 16:13:14,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 16:13:14,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 16:13:14,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 16:13:14,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 16:13:14,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 16:13:14,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 16:13:14,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 16:13:14,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 16:13:14,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 16:13:14,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 16:13:14,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 16:13:14,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 16:13:14,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 16:13:14,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 16:13:14,513 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 16:13:14,513 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 16:13:14,514 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 16:13:14,515 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T16:13:14,873 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487 2024-11-10 16:13:14,879 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 16:13:14,879 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T16:13:14,893 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T16:13:14,946 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T16:13:14,974 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c, deleteOnExit=true 2024-11-10T16:13:14,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T16:13:14,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/test.cache.data in system properties and HBase conf 2024-11-10T16:13:14,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T16:13:14,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.log.dir in system properties and HBase conf 2024-11-10T16:13:14,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T16:13:14,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T16:13:14,981 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T16:13:15,128 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T16:13:15,278 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T16:13:15,283 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T16:13:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T16:13:15,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T16:13:15,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T16:13:15,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T16:13:15,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T16:13:15,287 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T16:13:15,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T16:13:15,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T16:13:15,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/nfs.dump.dir in system properties and HBase conf 2024-11-10T16:13:15,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/java.io.tmpdir in system properties and HBase conf 2024-11-10T16:13:15,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T16:13:15,290 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T16:13:15,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T16:13:16,252 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T16:13:16,341 INFO [Time-limited test {}] log.Log(170): Logging initialized @2944ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T16:13:16,440 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T16:13:16,523 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T16:13:16,553 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T16:13:16,554 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T16:13:16,555 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T16:13:16,569 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T16:13:16,574 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.log.dir/,AVAILABLE} 2024-11-10T16:13:16,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T16:13:16,822 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/java.io.tmpdir/jetty-localhost-41935-hadoop-hdfs-3_4_1-tests_jar-_-any-11738441430406390564/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T16:13:16,831 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41935} 2024-11-10T16:13:16,832 INFO [Time-limited test {}] server.Server(415): Started @3436ms 2024-11-10T16:13:17,296 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T16:13:17,304 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T16:13:17,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T16:13:17,309 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T16:13:17,309 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T16:13:17,310 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.log.dir/,AVAILABLE} 2024-11-10T16:13:17,311 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T16:13:17,449 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/java.io.tmpdir/jetty-localhost-37083-hadoop-hdfs-3_4_1-tests_jar-_-any-3147106467971598602/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T16:13:17,450 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:37083} 2024-11-10T16:13:17,450 INFO [Time-limited test {}] server.Server(415): Started @4054ms 2024-11-10T16:13:17,509 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T16:13:18,175 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data2/current/BP-1564034578-172.17.0.2-1731255195956/current, will proceed with Du for space computation calculation, 2024-11-10T16:13:18,176 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data1/current/BP-1564034578-172.17.0.2-1731255195956/current, will proceed with Du for space computation calculation, 2024-11-10T16:13:18,252 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T16:13:18,323 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa875aa66e77f2acc with lease ID 0x68ff48d4479d123d: Processing first storage report for DS-b756177e-7265-49ee-bd93-f090e9cca49d from datanode DatanodeRegistration(127.0.0.1:37515, datanodeUuid=4bdfe523-a046-41fd-9c51-0ee6e262a8d8, infoPort=33179, infoSecurePort=0, ipcPort=34985, storageInfo=lv=-57;cid=testClusterID;nsid=602354352;c=1731255195956) 2024-11-10T16:13:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa875aa66e77f2acc with lease ID 0x68ff48d4479d123d: from storage DS-b756177e-7265-49ee-bd93-f090e9cca49d node DatanodeRegistration(127.0.0.1:37515, datanodeUuid=4bdfe523-a046-41fd-9c51-0ee6e262a8d8, infoPort=33179, infoSecurePort=0, ipcPort=34985, storageInfo=lv=-57;cid=testClusterID;nsid=602354352;c=1731255195956), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-11-10T16:13:18,327 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa875aa66e77f2acc with lease ID 0x68ff48d4479d123d: Processing first storage report for DS-6ac1ea11-9e6e-4013-9ae8-daacfca326b6 from datanode DatanodeRegistration(127.0.0.1:37515, datanodeUuid=4bdfe523-a046-41fd-9c51-0ee6e262a8d8, infoPort=33179, infoSecurePort=0, ipcPort=34985, storageInfo=lv=-57;cid=testClusterID;nsid=602354352;c=1731255195956) 2024-11-10T16:13:18,327 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa875aa66e77f2acc with lease ID 0x68ff48d4479d123d: from storage DS-6ac1ea11-9e6e-4013-9ae8-daacfca326b6 node DatanodeRegistration(127.0.0.1:37515, datanodeUuid=4bdfe523-a046-41fd-9c51-0ee6e262a8d8, infoPort=33179, infoSecurePort=0, ipcPort=34985, storageInfo=lv=-57;cid=testClusterID;nsid=602354352;c=1731255195956), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T16:13:18,366 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487 2024-11-10T16:13:18,480 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/zookeeper_0, clientPort=50949, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T16:13:18,491 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50949 2024-11-10T16:13:18,506 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:18,510 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:18,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741825_1001 (size=7) 2024-11-10T16:13:19,212 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 with version=8 2024-11-10T16:13:19,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/hbase-staging 2024-11-10T16:13:19,309 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T16:13:19,579 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8e3de9160f9f:0 server-side Connection retries=45 2024-11-10T16:13:19,590 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:19,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:19,596 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T16:13:19,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:19,597 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T16:13:19,761 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T16:13:19,824 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T16:13:19,837 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T16:13:19,843 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T16:13:19,882 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1012 (auto-detected) 2024-11-10T16:13:19,883 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T16:13:19,904 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41631 2024-11-10T16:13:19,931 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41631 connecting to ZooKeeper ensemble=127.0.0.1:50949 2024-11-10T16:13:19,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:416310x0, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T16:13:19,969 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41631-0x10102a7cd7a0000 connected 2024-11-10T16:13:19,999 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:20,001 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:20,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T16:13:20,022 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255, hbase.cluster.distributed=false 2024-11-10T16:13:20,046 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T16:13:20,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-10T16:13:20,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41631 2024-11-10T16:13:20,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41631 2024-11-10T16:13:20,058 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-10T16:13:20,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41631 2024-11-10T16:13:20,195 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8e3de9160f9f:0 server-side Connection retries=45 2024-11-10T16:13:20,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:20,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:20,198 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T16:13:20,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T16:13:20,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T16:13:20,201 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T16:13:20,204 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T16:13:20,205 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44105 2024-11-10T16:13:20,207 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44105 connecting to ZooKeeper ensemble=127.0.0.1:50949 2024-11-10T16:13:20,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:20,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:20,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:441050x0, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T16:13:20,222 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44105-0x10102a7cd7a0001 connected 2024-11-10T16:13:20,222 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T16:13:20,228 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T16:13:20,237 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T16:13:20,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T16:13:20,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T16:13:20,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44105 2024-11-10T16:13:20,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44105 2024-11-10T16:13:20,252 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44105 2024-11-10T16:13:20,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44105 2024-11-10T16:13:20,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44105 2024-11-10T16:13:20,287 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8e3de9160f9f:41631 2024-11-10T16:13:20,288 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:20,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T16:13:20,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T16:13:20,299 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:20,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T16:13:20,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:20,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:20,321 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T16:13:20,322 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8e3de9160f9f,41631,1731255199363 from backup master directory 2024-11-10T16:13:20,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:20,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T16:13:20,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T16:13:20,326 WARN [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T16:13:20,326 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:20,329 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T16:13:20,330 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T16:13:20,389 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/hbase.id] with ID: e8791941-1e0e-47f0-b506-1348b132a954 2024-11-10T16:13:20,390 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/.tmp/hbase.id 2024-11-10T16:13:20,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741826_1002 (size=42) 2024-11-10T16:13:20,802 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/.tmp/hbase.id]:[hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/hbase.id] 2024-11-10T16:13:20,859 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T16:13:20,866 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T16:13:20,891 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-10T16:13:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:20,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:20,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741827_1003 (size=196) 2024-11-10T16:13:21,328 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:13:21,331 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T16:13:21,354 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:21,360 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T16:13:21,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741828_1004 (size=1189) 2024-11-10T16:13:21,434 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store 2024-11-10T16:13:21,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741829_1005 (size=34) 2024-11-10T16:13:21,472 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T16:13:21,475 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:21,477 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T16:13:21,477 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:13:21,477 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:13:21,479 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T16:13:21,479 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:13:21,479 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:13:21,481 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731255201477Disabling compacts and flushes for region at 1731255201477Disabling writes for close at 1731255201479 (+2 ms)Writing region close event to WAL at 1731255201479Closed at 1731255201479 2024-11-10T16:13:21,483 WARN [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/.initializing 2024-11-10T16:13:21,483 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/WALs/8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:21,495 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T16:13:21,514 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C41631%2C1731255199363, suffix=, logDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/WALs/8e3de9160f9f,41631,1731255199363, archiveDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/oldWALs, maxLogs=10 2024-11-10T16:13:21,547 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/WALs/8e3de9160f9f,41631,1731255199363/8e3de9160f9f%2C41631%2C1731255199363.1731255201521, exclude list is [], retry=0 2024-11-10T16:13:21,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37515,DS-b756177e-7265-49ee-bd93-f090e9cca49d,DISK] 2024-11-10T16:13:21,568 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T16:13:21,608 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/WALs/8e3de9160f9f,41631,1731255199363/8e3de9160f9f%2C41631%2C1731255199363.1731255201521 2024-11-10T16:13:21,610 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33179:33179)] 2024-11-10T16:13:21,610 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:13:21,611 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:21,615 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,616 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,682 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T16:13:21,686 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:21,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:21,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T16:13:21,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:21,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:21,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T16:13:21,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:21,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:21,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,700 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T16:13:21,701 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:21,701 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:21,702 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,705 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,706 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,712 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,713 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,717 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T16:13:21,720 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T16:13:21,725 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:13:21,726 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64292513, jitterRate=-0.04196690022945404}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T16:13:21,731 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731255201629Initializing all the Stores at 1731255201631 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255201632 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255201633 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255201633Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255201633Cleaning up temporary data from old regions at 1731255201713 (+80 ms)Region opened successfully at 1731255201731 (+18 ms) 2024-11-10T16:13:21,732 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T16:13:21,769 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16a28d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8e3de9160f9f/172.17.0.2:0 2024-11-10T16:13:21,807 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T16:13:21,825 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T16:13:21,825 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T16:13:21,829 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T16:13:21,831 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T16:13:21,838 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-10T16:13:21,839 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T16:13:21,873 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T16:13:21,883 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T16:13:21,886 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T16:13:21,889 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T16:13:21,890 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T16:13:21,892 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T16:13:21,894 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T16:13:21,897 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T16:13:21,899 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T16:13:21,900 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T16:13:21,902 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T16:13:21,925 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T16:13:21,928 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T16:13:21,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T16:13:21,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T16:13:21,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,936 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8e3de9160f9f,41631,1731255199363, sessionid=0x10102a7cd7a0000, setting cluster-up flag (Was=false) 2024-11-10T16:13:21,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,958 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T16:13:21,960 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:21,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:21,973 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T16:13:21,975 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:21,982 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T16:13:22,066 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T16:13:22,071 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(746): ClusterId : e8791941-1e0e-47f0-b506-1348b132a954 2024-11-10T16:13:22,074 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T16:13:22,078 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T16:13:22,079 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T16:13:22,079 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T16:13:22,082 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T16:13:22,083 DEBUG [RS:0;8e3de9160f9f:44105 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f99645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8e3de9160f9f/172.17.0.2:0 2024-11-10T16:13:22,084 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T16:13:22,091 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8e3de9160f9f,41631,1731255199363 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T16:13:22,097 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8e3de9160f9f:44105 2024-11-10T16:13:22,098 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T16:13:22,098 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T16:13:22,098 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T16:13:22,098 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T16:13:22,098 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8e3de9160f9f:0, corePoolSize=10, maxPoolSize=10 2024-11-10T16:13:22,099 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,099 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T16:13:22,099 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,100 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731255232100 2024-11-10T16:13:22,100 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T16:13:22,101 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T16:13:22,101 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T16:13:22,102 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T16:13:22,103 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T16:13:22,103 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(2659): reportForDuty to master=8e3de9160f9f,41631,1731255199363 with port=44105, startcode=1731255200149 2024-11-10T16:13:22,105 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T16:13:22,106 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T16:13:22,106 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T16:13:22,107 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T16:13:22,107 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T16:13:22,108 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T16:13:22,108 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,112 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T16:13:22,113 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T16:13:22,113 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T16:13:22,114 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:22,115 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T16:13:22,115 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T16:13:22,115 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T16:13:22,117 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731255202116,5,FailOnTimeoutGroup] 2024-11-10T16:13:22,118 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731255202117,5,FailOnTimeoutGroup] 2024-11-10T16:13:22,118 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,118 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T16:13:22,118 DEBUG [RS:0;8e3de9160f9f:44105 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T16:13:22,119 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,119 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741831_1007 (size=1321) 2024-11-10T16:13:22,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46771, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T16:13:22,166 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41631 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,168 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41631 {}] master.ServerManager(517): Registering regionserver=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,183 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:13:22,184 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40533 2024-11-10T16:13:22,184 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T16:13:22,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T16:13:22,189 DEBUG [RS:0;8e3de9160f9f:44105 {}] zookeeper.ZKUtil(111): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,189 WARN [RS:0;8e3de9160f9f:44105 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T16:13:22,189 INFO [RS:0;8e3de9160f9f:44105 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T16:13:22,189 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,191 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8e3de9160f9f,44105,1731255200149] 2024-11-10T16:13:22,214 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T16:13:22,228 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T16:13:22,233 INFO [RS:0;8e3de9160f9f:44105 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T16:13:22,233 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,234 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T16:13:22,240 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T16:13:22,241 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,241 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8e3de9160f9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,242 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8e3de9160f9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8e3de9160f9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T16:13:22,243 DEBUG [RS:0;8e3de9160f9f:44105 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T16:13:22,244 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,245 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,245 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,245 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,245 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,245 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,44105,1731255200149-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T16:13:22,266 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T16:13:22,268 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,44105,1731255200149-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,269 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,269 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.Replication(171): 8e3de9160f9f,44105,1731255200149 started 2024-11-10T16:13:22,288 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:22,288 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1482): Serving as 8e3de9160f9f,44105,1731255200149, RpcServer on 8e3de9160f9f/172.17.0.2:44105, sessionid=0x10102a7cd7a0001 2024-11-10T16:13:22,289 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T16:13:22,289 DEBUG [RS:0;8e3de9160f9f:44105 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,289 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8e3de9160f9f,44105,1731255200149' 2024-11-10T16:13:22,289 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T16:13:22,290 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T16:13:22,291 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T16:13:22,291 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T16:13:22,291 DEBUG [RS:0;8e3de9160f9f:44105 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:22,291 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8e3de9160f9f,44105,1731255200149' 2024-11-10T16:13:22,291 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T16:13:22,292 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T16:13:22,293 DEBUG [RS:0;8e3de9160f9f:44105 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T16:13:22,293 INFO [RS:0;8e3de9160f9f:44105 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T16:13:22,293 INFO [RS:0;8e3de9160f9f:44105 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T16:13:22,398 INFO [RS:0;8e3de9160f9f:44105 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T16:13:22,402 INFO [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C44105%2C1731255200149, suffix=, logDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149, archiveDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/oldWALs, maxLogs=32 2024-11-10T16:13:22,418 DEBUG [RS:0;8e3de9160f9f:44105 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149/8e3de9160f9f%2C44105%2C1731255200149.1731255202404, exclude list is [], retry=0 2024-11-10T16:13:22,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37515,DS-b756177e-7265-49ee-bd93-f090e9cca49d,DISK] 2024-11-10T16:13:22,426 INFO [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149/8e3de9160f9f%2C44105%2C1731255200149.1731255202404 2024-11-10T16:13:22,427 DEBUG [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33179:33179)] 2024-11-10T16:13:22,530 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T16:13:22,531 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:13:22,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741833_1009 (size=32) 2024-11-10T16:13:22,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:22,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T16:13:22,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T16:13:22,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:22,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:22,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T16:13:22,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T16:13:22,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:22,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:22,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T16:13:22,955 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T16:13:22,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:22,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:22,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T16:13:22,959 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T16:13:22,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:22,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:22,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T16:13:22,961 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740 2024-11-10T16:13:22,962 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740 2024-11-10T16:13:22,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T16:13:22,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T16:13:22,965 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:13:22,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T16:13:22,971 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:13:22,972 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71936216, jitterRate=0.07193315029144287}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:13:22,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731255202942Initializing all the Stores at 1731255202944 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255202944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255202945 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255202945Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255202945Cleaning up temporary data from old regions at 1731255202965 (+20 ms)Region opened successfully at 1731255202974 (+9 ms) 2024-11-10T16:13:22,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T16:13:22,975 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T16:13:22,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T16:13:22,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T16:13:22,975 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T16:13:22,976 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T16:13:22,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731255202975Disabling compacts and flushes for region at 1731255202975Disabling writes for close at 1731255202975Writing region close event to WAL at 1731255202976 (+1 ms)Closed at 1731255202976 2024-11-10T16:13:22,980 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T16:13:22,980 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T16:13:22,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T16:13:22,995 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T16:13:22,997 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T16:13:23,150 DEBUG [8e3de9160f9f:41631 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T16:13:23,163 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:23,169 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8e3de9160f9f,44105,1731255200149, state=OPENING 2024-11-10T16:13:23,175 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T16:13:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:23,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:13:23,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T16:13:23,178 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T16:13:23,179 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T16:13:23,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:23,356 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T16:13:23,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53485, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T16:13:23,370 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T16:13:23,370 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T16:13:23,371 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T16:13:23,374 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8e3de9160f9f%2C44105%2C1731255200149.meta, suffix=.meta, logDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149, archiveDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/oldWALs, maxLogs=32 2024-11-10T16:13:23,391 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149/8e3de9160f9f%2C44105%2C1731255200149.meta.1731255203376.meta, exclude list is [], retry=0 2024-11-10T16:13:23,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37515,DS-b756177e-7265-49ee-bd93-f090e9cca49d,DISK] 2024-11-10T16:13:23,398 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/WALs/8e3de9160f9f,44105,1731255200149/8e3de9160f9f%2C44105%2C1731255200149.meta.1731255203376.meta 2024-11-10T16:13:23,398 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33179:33179)] 2024-11-10T16:13:23,398 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:13:23,400 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T16:13:23,403 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T16:13:23,407 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T16:13:23,412 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T16:13:23,412 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:23,413 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T16:13:23,413 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T16:13:23,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T16:13:23,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T16:13:23,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:23,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:23,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T16:13:23,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T16:13:23,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:23,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:23,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T16:13:23,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T16:13:23,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:23,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:23,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T16:13:23,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T16:13:23,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:23,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T16:13:23,427 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T16:13:23,428 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740 2024-11-10T16:13:23,431 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740 2024-11-10T16:13:23,433 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T16:13:23,433 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T16:13:23,434 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:13:23,437 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T16:13:23,439 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64624768, jitterRate=-0.03701591491699219}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:13:23,439 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T16:13:23,441 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731255203413Writing region info on filesystem at 1731255203413Initializing all the Stores at 1731255203415 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255203416 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255203416Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255203416Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731255203416Cleaning up temporary data from old regions at 1731255203433 (+17 ms)Running coprocessor post-open hooks at 1731255203439 (+6 ms)Region opened successfully at 1731255203441 (+2 ms) 2024-11-10T16:13:23,448 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731255203347 2024-11-10T16:13:23,461 DEBUG [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T16:13:23,462 INFO [RS_OPEN_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T16:13:23,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:23,465 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8e3de9160f9f,44105,1731255200149, state=OPEN 2024-11-10T16:13:23,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T16:13:23,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T16:13:23,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T16:13:23,477 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T16:13:23,477 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:23,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T16:13:23,483 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8e3de9160f9f,44105,1731255200149 in 297 msec 2024-11-10T16:13:23,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T16:13:23,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 498 msec 2024-11-10T16:13:23,491 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T16:13:23,491 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T16:13:23,513 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:23,514 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:23,537 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:23,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:23,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5500 sec 2024-11-10T16:13:23,565 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731255203565, completionTime=-1 2024-11-10T16:13:23,567 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T16:13:23,567 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T16:13:23,598 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T16:13:23,598 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731255263598 2024-11-10T16:13:23,598 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731255323598 2024-11-10T16:13:23,599 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 31 msec 2024-11-10T16:13:23,601 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,601 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,601 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,603 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8e3de9160f9f:41631, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,604 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,605 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,610 DEBUG [master/8e3de9160f9f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T16:13:23,632 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.306sec 2024-11-10T16:13:23,633 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T16:13:23,635 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T16:13:23,636 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T16:13:23,636 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T16:13:23,636 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T16:13:23,637 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T16:13:23,638 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T16:13:23,646 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T16:13:23,647 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T16:13:23,648 INFO [master/8e3de9160f9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8e3de9160f9f,41631,1731255199363-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T16:13:23,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:23,684 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T16:13:23,684 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T16:13:23,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:23,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:23,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f86640e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:23,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:23,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:23,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:23,706 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:23,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:23,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:23,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:23,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:23,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:23,753 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=339, ProcessCount=11, AvailableMemoryMB=7728 2024-11-10T16:13:23,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:13:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:23,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@713ddca7 2024-11-10T16:13:23,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:13:23,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:13:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:13:23,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:13:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:23,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:13:23,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T16:13:23,795 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:23,797 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:13:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T16:13:23,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741835_1011 (size=1061) 2024-11-10T16:13:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T16:13:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T16:13:24,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:13:24,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741836_1012 (size=53) 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c8fb0b94c7d60b2a8bd105a3337e427b, disabling compactions & flushes 2024-11-10T16:13:24,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. after waiting 0 ms 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8fb0b94c7d60b2a8bd105a3337e427b: Waiting for close lock at 1731255204224Disabling compacts and flushes for region at 1731255204224Disabling writes for close at 1731255204224Writing region close event to WAL at 1731255204224Closed at 1731255204224 2024-11-10T16:13:24,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:13:24,232 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255204227"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255204227"}]},"ts":"1731255204227"} 2024-11-10T16:13:24,237 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:13:24,239 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:13:24,242 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255204239"}]},"ts":"1731255204239"} 2024-11-10T16:13:24,247 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:13:24,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, ASSIGN}] 2024-11-10T16:13:24,263 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, ASSIGN 2024-11-10T16:13:24,265 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:13:24,416 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8fb0b94c7d60b2a8bd105a3337e427b, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:24,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, ASSIGN because future has completed 2024-11-10T16:13:24,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T16:13:24,587 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,588 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:13:24,588 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,588 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:24,588 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,588 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,591 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,594 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:24,595 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8fb0b94c7d60b2a8bd105a3337e427b columnFamilyName A 2024-11-10T16:13:24,595 DEBUG [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:24,596 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(327): Store=c8fb0b94c7d60b2a8bd105a3337e427b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:24,596 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,598 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:24,598 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8fb0b94c7d60b2a8bd105a3337e427b columnFamilyName B 2024-11-10T16:13:24,598 DEBUG [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:24,599 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(327): Store=c8fb0b94c7d60b2a8bd105a3337e427b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:24,599 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,601 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:24,601 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8fb0b94c7d60b2a8bd105a3337e427b columnFamilyName C 2024-11-10T16:13:24,601 DEBUG [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:24,602 INFO [StoreOpener-c8fb0b94c7d60b2a8bd105a3337e427b-1 {}] regionserver.HStore(327): Store=c8fb0b94c7d60b2a8bd105a3337e427b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:24,603 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,603 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,604 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,605 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,607 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,607 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,608 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:13:24,610 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,614 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:13:24,615 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c8fb0b94c7d60b2a8bd105a3337e427b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71188736, jitterRate=0.060794830322265625}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:13:24,615 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:24,616 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c8fb0b94c7d60b2a8bd105a3337e427b: Running coprocessor pre-open hook at 1731255204589Writing region info on filesystem at 1731255204589Initializing all the Stores at 1731255204590 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255204590Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255204590Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255204591 (+1 ms)Cleaning up temporary data from old regions at 1731255204607 (+16 ms)Running coprocessor post-open hooks at 1731255204615 (+8 ms)Region opened successfully at 1731255204616 (+1 ms) 2024-11-10T16:13:24,618 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., pid=6, masterSystemTime=1731255204580 2024-11-10T16:13:24,622 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,622 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:24,624 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8fb0b94c7d60b2a8bd105a3337e427b, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:24,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:13:24,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T16:13:24,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 in 203 msec 2024-11-10T16:13:24,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T16:13:24,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, ASSIGN in 375 msec 2024-11-10T16:13:24,639 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:13:24,639 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255204639"}]},"ts":"1731255204639"} 2024-11-10T16:13:24,643 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:13:24,645 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:13:24,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 862 msec 2024-11-10T16:13:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T16:13:24,944 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:24,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:24,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:24,952 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eb5e284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:24,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:24,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17282950, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,957 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:24,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:24,961 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:24,964 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:13:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:24,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24720917 2024-11-10T16:13:24,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:13:24,969 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:13:24,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:24,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:24,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:24,978 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10bc6ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:24,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:24,982 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e153ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:24,985 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64257f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:24,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:24,988 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:24,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b37f010, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:24,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:24,993 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:24,994 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643162ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:24,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:24,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:24,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e855a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,000 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b24d866, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,001 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:25,001 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:25,002 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3964ad25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,004 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59420, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ea0df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:25,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:25,007 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf4dbe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:25,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:25,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,010 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6a7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:25,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:25,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,015 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ba8772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:25,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:25,019 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4686c825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,022 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f254c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,024 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:25,024 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:25,025 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441ebf29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:25,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:25,027 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b6ade8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:25,033 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,033 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,034 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,034 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,034 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,037 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,041 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:25,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,050 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T16:13:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,055 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,057 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:25,059 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:25,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:25,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:25,064 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:25,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:25,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:25,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:13:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:25,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:13:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:25,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:25,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:25,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:25,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:25,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:25,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:25,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/48a2ff327be7481db556054c0957113f is 50, key is test_row_0/A:col10/1731255205058/Put/seqid=0 2024-11-10T16:13:25,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741837_1013 (size=12001) 2024-11-10T16:13:25,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/48a2ff327be7481db556054c0957113f 2024-11-10T16:13:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255265245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,296 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731255205202, max timestamp=1731255205280Num uniques 859; 2024-11-10T16:13:25,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T16:13:25,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255205227, max timestamp=1731255205280Num uniques 859; 2024-11-10T16:13:25,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255205227, max timestamp=1731255205298Num uniques 859; 2024-11-10T16:13:25,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7688, suffix heap size=117064,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=168 2024-11-10T16:13:25,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:13:25,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255265364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255265366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255265379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255265384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/538cefe891534acca135215665635d9a is 50, key is test_row_0/B:col10/1731255205058/Put/seqid=0 2024-11-10T16:13:25,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:25,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255265435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741838_1014 (size=12001) 2024-11-10T16:13:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/538cefe891534acca135215665635d9a 2024-11-10T16:13:25,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/9d4a19ec55674956b88791507a411a51 is 50, key is test_row_0/C:col10/1731255205058/Put/seqid=0 2024-11-10T16:13:25,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255265494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255265504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255265494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255265505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741839_1015 (size=12001) 2024-11-10T16:13:25,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/9d4a19ec55674956b88791507a411a51 2024-11-10T16:13:25,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/48a2ff327be7481db556054c0957113f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f 2024-11-10T16:13:25,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T16:13:25,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/538cefe891534acca135215665635d9a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a 2024-11-10T16:13:25,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T16:13:25,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/9d4a19ec55674956b88791507a411a51 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51 2024-11-10T16:13:25,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:25,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51, entries=150, sequenceid=13, filesize=11.7 K 2024-11-10T16:13:25,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=121.17 KB/124074 for c8fb0b94c7d60b2a8bd105a3337e427b in 476ms, sequenceid=13, compaction requested=false 2024-11-10T16:13:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T16:13:25,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:25,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=127.88 KB heapSize=361.29 KB 2024-11-10T16:13:25,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:25,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:25,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:25,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:25,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:25,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:25,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/81e4f9e0a6a343deb2332dc37379de4e is 50, key is test_row_0/A:col10/1731255205655/Put/seqid=0 2024-11-10T16:13:25,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255265733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255265738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741840_1016 (size=12001) 2024-11-10T16:13:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/81e4f9e0a6a343deb2332dc37379de4e 2024-11-10T16:13:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255265745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255265748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255265748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cec06e5889f54760b113dafcc1b93941 is 50, key is test_row_0/B:col10/1731255205655/Put/seqid=0 2024-11-10T16:13:25,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741841_1017 (size=12001) 2024-11-10T16:13:25,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.66 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cec06e5889f54760b113dafcc1b93941 2024-11-10T16:13:25,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/3ae05e33fea24195b8bb23bf9a725b36 is 50, key is test_row_0/C:col10/1731255205655/Put/seqid=0 2024-11-10T16:13:25,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255265854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255265855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741842_1018 (size=12001) 2024-11-10T16:13:25,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255265876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255265876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255265884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:25,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:25,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:25,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:25,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:25,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255266064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255266074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255266085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255266086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255266104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:26,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:26,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/3ae05e33fea24195b8bb23bf9a725b36 2024-11-10T16:13:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/81e4f9e0a6a343deb2332dc37379de4e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e 2024-11-10T16:13:26,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T16:13:26,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cec06e5889f54760b113dafcc1b93941 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941 2024-11-10T16:13:26,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T16:13:26,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/3ae05e33fea24195b8bb23bf9a725b36 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36 2024-11-10T16:13:26,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T16:13:26,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.88 KB/130944, heapSize ~361.24 KB/369912, currentSize=53.67 KB/54960 for c8fb0b94c7d60b2a8bd105a3337e427b in 691ms, sequenceid=48, compaction requested=false 2024-11-10T16:13:26,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:26,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T16:13:26,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,390 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:13:26,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:26,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:26,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:26,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/ed2b1607b3ba4a31983fbde3918f4c8d is 50, key is test_row_0/A:col10/1731255206390/Put/seqid=0 2024-11-10T16:13:26,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741843_1019 (size=16681) 2024-11-10T16:13:26,440 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/ed2b1607b3ba4a31983fbde3918f4c8d 2024-11-10T16:13:26,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/1897ebdd37364289b5ae43dbad910b1d is 50, key is test_row_0/B:col10/1731255206390/Put/seqid=0 2024-11-10T16:13:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255266467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255266473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255266473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255266474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255266477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741844_1020 (size=12001) 2024-11-10T16:13:26,512 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/1897ebdd37364289b5ae43dbad910b1d 2024-11-10T16:13:26,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/5ee0026a67564a90be0e6327033f427a is 50, key is test_row_0/C:col10/1731255206390/Put/seqid=0 2024-11-10T16:13:26,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741845_1021 (size=12001) 2024-11-10T16:13:26,565 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/5ee0026a67564a90be0e6327033f427a 2024-11-10T16:13:26,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/ed2b1607b3ba4a31983fbde3918f4c8d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d 2024-11-10T16:13:26,591 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d, entries=250, sequenceid=61, filesize=16.3 K 2024-11-10T16:13:26,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/1897ebdd37364289b5ae43dbad910b1d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d 2024-11-10T16:13:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255266594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255266594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d, entries=150, sequenceid=61, filesize=11.7 K 2024-11-10T16:13:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/5ee0026a67564a90be0e6327033f427a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a 2024-11-10T16:13:26,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255266595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255266595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255266596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,619 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-10T16:13:26,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c8fb0b94c7d60b2a8bd105a3337e427b in 231ms, sequenceid=61, compaction requested=true 2024-11-10T16:13:26,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:26,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T16:13:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T16:13:26,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T16:13:26,636 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-10T16:13:26,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5940 sec 2024-11-10T16:13:26,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:13:26,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:26,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255206442, max timestamp=1731255206817Num uniques 859; 2024-11-10T16:13:26,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:13:26,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:26,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255206442, max timestamp=1731255206817Num uniques 859; 2024-11-10T16:13:26,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:26,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:26,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:26,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:26,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:26,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:13:26,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f0cae60e095e4eb989cd655c1997888a is 50, key is test_row_0/A:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741846_1022 (size=12001) 2024-11-10T16:13:26,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f0cae60e095e4eb989cd655c1997888a 2024-11-10T16:13:26,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/f49b02b560d745b2a623baeb69250ba2 is 50, key is test_row_0/B:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:26,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741847_1023 (size=12001) 2024-11-10T16:13:26,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/f49b02b560d745b2a623baeb69250ba2 2024-11-10T16:13:26,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c815b9136b4940fbadfa5a556547306d is 50, key is test_row_0/C:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:26,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255266905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255266907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255266911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255266912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255266915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:26,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:26,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741848_1024 (size=12001) 2024-11-10T16:13:26,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c815b9136b4940fbadfa5a556547306d 2024-11-10T16:13:26,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f0cae60e095e4eb989cd655c1997888a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a 2024-11-10T16:13:26,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a, entries=150, sequenceid=84, filesize=11.7 K 2024-11-10T16:13:26,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/f49b02b560d745b2a623baeb69250ba2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2 2024-11-10T16:13:26,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2, entries=150, sequenceid=84, filesize=11.7 K 2024-11-10T16:13:26,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c815b9136b4940fbadfa5a556547306d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d 2024-11-10T16:13:27,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d, entries=150, sequenceid=84, filesize=11.7 K 2024-11-10T16:13:27,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~203.97 KB/208864, currentSize=107.34 KB/109920 for c8fb0b94c7d60b2a8bd105a3337e427b in 184ms, sequenceid=84, compaction requested=true 2024-11-10T16:13:27,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:27,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:27,018 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:27,018 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:27,023 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:27,025 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:27,025 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:27,025 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:27,026 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,026 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,027 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=51.4 K 2024-11-10T16:13:27,027 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=46.9 K 2024-11-10T16:13:27,028 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 538cefe891534acca135215665635d9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731255205058 2024-11-10T16:13:27,028 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a2ff327be7481db556054c0957113f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731255205058 2024-11-10T16:13:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:27,029 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting cec06e5889f54760b113dafcc1b93941, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731255205361 2024-11-10T16:13:27,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:27,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,030 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1897ebdd37364289b5ae43dbad910b1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731255205738 2024-11-10T16:13:27,031 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f49b02b560d745b2a623baeb69250ba2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:27,032 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e4f9e0a6a343deb2332dc37379de4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731255205361 2024-11-10T16:13:27,035 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2b1607b3ba4a31983fbde3918f4c8d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731255205732 2024-11-10T16:13:27,038 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0cae60e095e4eb989cd655c1997888a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:27,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c4c14f6531ed471ebb4e3ae45c7e6b15 is 50, key is test_row_0/A:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:27,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255267054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255267066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255267069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255267070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255267079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,099 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#13 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:27,100 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/84306a79008542beba8e490d80eb6404 is 50, key is test_row_0/B:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:27,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741849_1025 (size=14341) 2024-11-10T16:13:27,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c4c14f6531ed471ebb4e3ae45c7e6b15 2024-11-10T16:13:27,120 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#14 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:27,121 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c77831f186c54b1daad860d1ee6a9a5e is 50, key is test_row_0/A:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:27,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741850_1026 (size=12139) 2024-11-10T16:13:27,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/8f80a98d6b034f52b88d77a2aeeaa247 is 50, key is test_row_0/B:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:27,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741851_1027 (size=12139) 2024-11-10T16:13:27,156 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c77831f186c54b1daad860d1ee6a9a5e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c77831f186c54b1daad860d1ee6a9a5e 2024-11-10T16:13:27,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741852_1028 (size=12001) 2024-11-10T16:13:27,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/8f80a98d6b034f52b88d77a2aeeaa247 2024-11-10T16:13:27,190 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into c77831f186c54b1daad860d1ee6a9a5e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:27,191 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255267184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,197 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=12, startTime=1731255207011; duration=0sec 2024-11-10T16:13:27,198 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:27,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/521d0fa03ac244cc8433ae293af981e4 is 50, key is test_row_0/C:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:27,198 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:27,198 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:27,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,203 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:27,203 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:27,203 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,204 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=46.9 K 2024-11-10T16:13:27,205 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d4a19ec55674956b88791507a411a51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731255205058 2024-11-10T16:13:27,206 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae05e33fea24195b8bb23bf9a725b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731255205361 2024-11-10T16:13:27,207 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ee0026a67564a90be0e6327033f427a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731255205738 2024-11-10T16:13:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255267195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,209 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c815b9136b4940fbadfa5a556547306d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255267195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255267201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255267196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T16:13:27,218 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:27,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T16:13:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T16:13:27,226 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:27,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:27,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:27,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741853_1029 (size=12001) 2024-11-10T16:13:27,264 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#17 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:27,265 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6a6a190cc1f24cdcad5bccaca7b38b92 is 50, key is test_row_0/C:col10/1731255206442/Put/seqid=0 2024-11-10T16:13:27,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741854_1030 (size=12139) 2024-11-10T16:13:27,324 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6a6a190cc1f24cdcad5bccaca7b38b92 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6a6a190cc1f24cdcad5bccaca7b38b92 2024-11-10T16:13:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T16:13:27,340 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 6a6a190cc1f24cdcad5bccaca7b38b92(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:27,340 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,340 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=12, startTime=1731255207015; duration=0sec 2024-11-10T16:13:27,341 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,341 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:27,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T16:13:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255267415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255267415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255267415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255267424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255267424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T16:13:27,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:27,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T16:13:27,551 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/84306a79008542beba8e490d80eb6404 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/84306a79008542beba8e490d80eb6404 2024-11-10T16:13:27,570 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 84306a79008542beba8e490d80eb6404(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:27,570 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,570 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=12, startTime=1731255207015; duration=0sec 2024-11-10T16:13:27,570 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,570 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:27,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/521d0fa03ac244cc8433ae293af981e4 2024-11-10T16:13:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c4c14f6531ed471ebb4e3ae45c7e6b15 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15 2024-11-10T16:13:27,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15, entries=200, sequenceid=104, filesize=14.0 K 2024-11-10T16:13:27,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/8f80a98d6b034f52b88d77a2aeeaa247 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247 2024-11-10T16:13:27,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T16:13:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:27,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T16:13:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/521d0fa03ac244cc8433ae293af981e4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4 2024-11-10T16:13:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4, entries=150, sequenceid=104, filesize=11.7 K 2024-11-10T16:13:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255267723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255267723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c8fb0b94c7d60b2a8bd105a3337e427b in 702ms, sequenceid=104, compaction requested=false 2024-11-10T16:13:27,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:27,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:27,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/dea8dbea4f394db38219f64a9ec841f4 is 50, key is test_row_1/A:col10/1731255207729/Put/seqid=0 2024-11-10T16:13:27,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741855_1031 (size=9657) 2024-11-10T16:13:27,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/dea8dbea4f394db38219f64a9ec841f4 2024-11-10T16:13:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255267785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255267785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255267795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/5a1a2d5ce66146629e8881ea787431fc is 50, key is test_row_1/B:col10/1731255207729/Put/seqid=0 2024-11-10T16:13:27,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741856_1032 (size=9657) 2024-11-10T16:13:27,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/5a1a2d5ce66146629e8881ea787431fc 2024-11-10T16:13:27,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/758950887f16461faa76242f90636582 is 50, key is test_row_1/C:col10/1731255207729/Put/seqid=0 2024-11-10T16:13:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T16:13:27,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T16:13:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741857_1033 (size=9657) 2024-11-10T16:13:27,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/758950887f16461faa76242f90636582 2024-11-10T16:13:27,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/dea8dbea4f394db38219f64a9ec841f4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4 2024-11-10T16:13:27,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4, entries=100, sequenceid=120, filesize=9.4 K 2024-11-10T16:13:27,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255267903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/5a1a2d5ce66146629e8881ea787431fc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc 2024-11-10T16:13:27,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255267904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255267914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:27,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:27,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:27,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc, entries=100, sequenceid=120, filesize=9.4 K 2024-11-10T16:13:27,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/758950887f16461faa76242f90636582 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582 2024-11-10T16:13:27,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582, entries=100, sequenceid=120, filesize=9.4 K 2024-11-10T16:13:27,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c8fb0b94c7d60b2a8bd105a3337e427b in 206ms, sequenceid=120, compaction requested=true 2024-11-10T16:13:27,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:27,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:27,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,939 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:27,940 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:27,943 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:27,943 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:27,943 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,944 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c77831f186c54b1daad860d1ee6a9a5e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=35.3 K 2024-11-10T16:13:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:27,945 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:27,945 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:27,945 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:27,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:27,946 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/84306a79008542beba8e490d80eb6404, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=33.0 K 2024-11-10T16:13:27,946 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c77831f186c54b1daad860d1ee6a9a5e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:27,946 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 84306a79008542beba8e490d80eb6404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:27,947 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c14f6531ed471ebb4e3ae45c7e6b15, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731255206848 2024-11-10T16:13:27,947 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f80a98d6b034f52b88d77a2aeeaa247, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731255206899 2024-11-10T16:13:27,948 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting dea8dbea4f394db38219f64a9ec841f4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255207060 2024-11-10T16:13:27,948 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a1a2d5ce66146629e8881ea787431fc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255207060 2024-11-10T16:13:27,975 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:27,976 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2e44ad63c25a4f32a3927fd821c2748d is 50, key is test_row_0/A:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:27,987 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:27,987 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/e427fae697c84dcd8d3db17f5cabffa4 is 50, key is test_row_0/B:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:28,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741858_1034 (size=12241) 2024-11-10T16:13:28,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T16:13:28,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,015 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:28,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741859_1035 (size=12241) 2024-11-10T16:13:28,038 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/e427fae697c84dcd8d3db17f5cabffa4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e427fae697c84dcd8d3db17f5cabffa4 2024-11-10T16:13:28,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2dacc0c8a2504e6f86df745112427110 is 50, key is test_row_0/A:col10/1731255207777/Put/seqid=0 2024-11-10T16:13:28,057 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into e427fae697c84dcd8d3db17f5cabffa4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:28,057 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,057 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255207939; duration=0sec 2024-11-10T16:13:28,058 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:28,058 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:28,060 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:28,062 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:28,062 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:28,063 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,063 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6a6a190cc1f24cdcad5bccaca7b38b92, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=33.0 K 2024-11-10T16:13:28,067 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6a190cc1f24cdcad5bccaca7b38b92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731255206442 2024-11-10T16:13:28,068 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 521d0fa03ac244cc8433ae293af981e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731255206899 2024-11-10T16:13:28,069 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 758950887f16461faa76242f90636582, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255207060 2024-11-10T16:13:28,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741860_1036 (size=12151) 2024-11-10T16:13:28,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2dacc0c8a2504e6f86df745112427110 2024-11-10T16:13:28,101 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:28,102 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/9b6ff284e7fa494a980d216066793e15 is 50, key is test_row_0/C:col10/1731255206899/Put/seqid=0 2024-11-10T16:13:28,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/ee92a610529f4e4c9075f07c692b1af9 is 50, key is test_row_0/B:col10/1731255207777/Put/seqid=0 2024-11-10T16:13:28,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741861_1037 (size=12241) 2024-11-10T16:13:28,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:28,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/9b6ff284e7fa494a980d216066793e15 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9b6ff284e7fa494a980d216066793e15 2024-11-10T16:13:28,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741862_1038 (size=12151) 2024-11-10T16:13:28,144 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/ee92a610529f4e4c9075f07c692b1af9 2024-11-10T16:13:28,155 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 9b6ff284e7fa494a980d216066793e15(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:28,157 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,157 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255207945; duration=0sec 2024-11-10T16:13:28,157 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:28,157 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:28,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/26016c9fa8e74296a6e97dcc5616d925 is 50, key is test_row_0/C:col10/1731255207777/Put/seqid=0 2024-11-10T16:13:28,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255268158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255268165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255268164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741863_1039 (size=12151) 2024-11-10T16:13:28,181 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/26016c9fa8e74296a6e97dcc5616d925 2024-11-10T16:13:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2dacc0c8a2504e6f86df745112427110 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110 2024-11-10T16:13:28,207 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T16:13:28,208 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T16:13:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/ee92a610529f4e4c9075f07c692b1af9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9 2024-11-10T16:13:28,220 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T16:13:28,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/26016c9fa8e74296a6e97dcc5616d925 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925 2024-11-10T16:13:28,235 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T16:13:28,240 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c8fb0b94c7d60b2a8bd105a3337e427b in 224ms, sequenceid=139, compaction requested=false 2024-11-10T16:13:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T16:13:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T16:13:28,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:28,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T16:13:28,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-10T16:13:28,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0360 sec 2024-11-10T16:13:28,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a03c1a47a69144ad86c8f01068d5ce46 is 50, key is test_row_0/A:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741864_1040 (size=16931) 2024-11-10T16:13:28,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a03c1a47a69144ad86c8f01068d5ce46 2024-11-10T16:13:28,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255268303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/362cfc33fb474ee6a1b31ec1e738d48e is 50, key is test_row_0/B:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255268306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255268307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255268309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255268318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741865_1041 (size=12151) 2024-11-10T16:13:28,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/362cfc33fb474ee6a1b31ec1e738d48e 2024-11-10T16:13:28,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c0f23d655c1a45df950358bd553e8238 is 50, key is test_row_0/C:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T16:13:28,363 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:28,364 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T16:13:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T16:13:28,369 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:28,371 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:28,372 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:28,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741866_1042 (size=12151) 2024-11-10T16:13:28,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c0f23d655c1a45df950358bd553e8238 2024-11-10T16:13:28,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a03c1a47a69144ad86c8f01068d5ce46 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46 2024-11-10T16:13:28,425 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2e44ad63c25a4f32a3927fd821c2748d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2e44ad63c25a4f32a3927fd821c2748d 2024-11-10T16:13:28,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255268423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46, entries=250, sequenceid=156, filesize=16.5 K 2024-11-10T16:13:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/362cfc33fb474ee6a1b31ec1e738d48e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e 2024-11-10T16:13:28,438 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 2e44ad63c25a4f32a3927fd821c2748d(size=12.0 K), total size for store is 40.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:28,438 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,438 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255207939; duration=0sec 2024-11-10T16:13:28,439 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:28,439 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:28,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255268434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T16:13:28,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255268444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c0f23d655c1a45df950358bd553e8238 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238 2024-11-10T16:13:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255268445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255268445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T16:13:28,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c8fb0b94c7d60b2a8bd105a3337e427b in 223ms, sequenceid=156, compaction requested=true 2024-11-10T16:13:28,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:28,465 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:28,465 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:28,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:28,472 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:28,473 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:28,473 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,473 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41323 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:28,473 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:28,473 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e427fae697c84dcd8d3db17f5cabffa4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=35.7 K 2024-11-10T16:13:28,473 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,473 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2e44ad63c25a4f32a3927fd821c2748d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=40.4 K 2024-11-10T16:13:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T16:13:28,474 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e427fae697c84dcd8d3db17f5cabffa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255206899 2024-11-10T16:13:28,474 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e44ad63c25a4f32a3927fd821c2748d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255206899 2024-11-10T16:13:28,475 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting ee92a610529f4e4c9075f07c692b1af9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255207777 2024-11-10T16:13:28,475 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dacc0c8a2504e6f86df745112427110, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255207777 2024-11-10T16:13:28,476 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 362cfc33fb474ee6a1b31ec1e738d48e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:28,476 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a03c1a47a69144ad86c8f01068d5ce46, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:28,499 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:28,500 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/9f122800fd2141619db55416d14da53c is 50, key is test_row_0/B:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,504 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#31 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:28,505 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/fa9efac19aea4bafbc62d298f0545279 is 50, key is test_row_0/A:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741868_1044 (size=12493) 2024-11-10T16:13:28,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T16:13:28,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,529 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:13:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d6d1ce4141a34287a9b1e9c75120e536 is 50, key is test_row_0/A:col10/1731255208297/Put/seqid=0 2024-11-10T16:13:28,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741867_1043 (size=12493) 2024-11-10T16:13:28,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741869_1045 (size=12151) 2024-11-10T16:13:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:28,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T16:13:28,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255268678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255268680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255268680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255268686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255268688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255268793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255268793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255268794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255268803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255268803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:28,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:28,935 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/fa9efac19aea4bafbc62d298f0545279 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/fa9efac19aea4bafbc62d298f0545279 2024-11-10T16:13:28,938 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:13:28,952 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into fa9efac19aea4bafbc62d298f0545279(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:28,952 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,952 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255208465; duration=0sec 2024-11-10T16:13:28,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:28,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:28,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:28,955 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:28,955 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:28,955 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:28,957 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9b6ff284e7fa494a980d216066793e15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=35.7 K 2024-11-10T16:13:28,959 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6ff284e7fa494a980d216066793e15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731255206899 2024-11-10T16:13:28,960 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26016c9fa8e74296a6e97dcc5616d925, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255207777 2024-11-10T16:13:28,962 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f23d655c1a45df950358bd553e8238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:28,965 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d6d1ce4141a34287a9b1e9c75120e536 2024-11-10T16:13:28,973 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/9f122800fd2141619db55416d14da53c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/9f122800fd2141619db55416d14da53c 2024-11-10T16:13:28,990 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 9f122800fd2141619db55416d14da53c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:28,990 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:28,990 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255208465; duration=0sec 2024-11-10T16:13:28,991 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:28,991 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:28,991 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:28,992 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/458b3edf4f1842b485459139ed292abf is 50, key is test_row_0/C:col10/1731255208153/Put/seqid=0 2024-11-10T16:13:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T16:13:28,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/e902e10e767441489a00ea7e24ff1561 is 50, key is test_row_0/B:col10/1731255208297/Put/seqid=0 2024-11-10T16:13:29,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255269004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255269014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255269013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255269014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255269015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741870_1046 (size=12151) 2024-11-10T16:13:29,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741871_1047 (size=12493) 2024-11-10T16:13:29,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/e902e10e767441489a00ea7e24ff1561 2024-11-10T16:13:29,055 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/458b3edf4f1842b485459139ed292abf as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/458b3edf4f1842b485459139ed292abf 2024-11-10T16:13:29,069 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 458b3edf4f1842b485459139ed292abf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:29,069 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:29,070 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255208465; duration=0sec 2024-11-10T16:13:29,070 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:29,071 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:29,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6adbb23a171b4044b1f090956e3973d7 is 50, key is test_row_0/C:col10/1731255208297/Put/seqid=0 2024-11-10T16:13:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741872_1048 (size=12151) 2024-11-10T16:13:29,093 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6adbb23a171b4044b1f090956e3973d7 2024-11-10T16:13:29,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d6d1ce4141a34287a9b1e9c75120e536 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536 2024-11-10T16:13:29,115 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536, entries=150, sequenceid=174, filesize=11.9 K 2024-11-10T16:13:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/e902e10e767441489a00ea7e24ff1561 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561 2024-11-10T16:13:29,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561, entries=150, sequenceid=174, filesize=11.9 K 2024-11-10T16:13:29,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6adbb23a171b4044b1f090956e3973d7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7 2024-11-10T16:13:29,142 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7, entries=150, sequenceid=174, filesize=11.9 K 2024-11-10T16:13:29,145 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c8fb0b94c7d60b2a8bd105a3337e427b in 616ms, sequenceid=174, compaction requested=false 2024-11-10T16:13:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T16:13:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T16:13:29,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T16:13:29,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 775 msec 2024-11-10T16:13:29,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 788 msec 2024-11-10T16:13:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:29,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:13:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:29,318 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2d181c8842d044b993edbd882ba5427f is 50, key is test_row_0/A:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:29,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741873_1049 (size=14541) 2024-11-10T16:13:29,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2d181c8842d044b993edbd882ba5427f 2024-11-10T16:13:29,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255269348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255269355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255269354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255269356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255269359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7a5c5476075a483da9e1936fc487669c is 50, key is test_row_0/B:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:29,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741874_1050 (size=12151) 2024-11-10T16:13:29,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7a5c5476075a483da9e1936fc487669c 2024-11-10T16:13:29,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f49de7a597564ea1beda72ff37f19b94 is 50, key is test_row_0/C:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:29,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741875_1051 (size=12151) 2024-11-10T16:13:29,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255269464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255269464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255269473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255269474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255269474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T16:13:29,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:29,506 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T16:13:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T16:13:29,509 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:29,511 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:29,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T16:13:29,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T16:13:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:29,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255269674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255269674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255269683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255269684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255269693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:29,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:29,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T16:13:29,820 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T16:13:29,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:29,820 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T16:13:29,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:29,823 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T16:13:29,823 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T16:13:29,823 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T16:13:29,823 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T16:13:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T16:13:29,828 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T16:13:29,828 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T16:13:29,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f49de7a597564ea1beda72ff37f19b94 2024-11-10T16:13:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/2d181c8842d044b993edbd882ba5427f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f 2024-11-10T16:13:29,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f, entries=200, sequenceid=193, filesize=14.2 K 2024-11-10T16:13:29,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7a5c5476075a483da9e1936fc487669c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c 2024-11-10T16:13:29,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T16:13:29,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f49de7a597564ea1beda72ff37f19b94 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94 2024-11-10T16:13:29,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T16:13:29,892 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T16:13:29,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c8fb0b94c7d60b2a8bd105a3337e427b in 578ms, sequenceid=193, compaction requested=true 2024-11-10T16:13:29,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:29,895 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:29,895 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:29,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:29,898 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:29,898 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:29,898 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:29,898 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:29,898 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,898 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,899 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/fa9efac19aea4bafbc62d298f0545279, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=38.3 K 2024-11-10T16:13:29,899 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/9f122800fd2141619db55416d14da53c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=35.9 K 2024-11-10T16:13:29,900 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9efac19aea4bafbc62d298f0545279, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:29,900 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f122800fd2141619db55416d14da53c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:29,901 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6d1ce4141a34287a9b1e9c75120e536, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255208297 2024-11-10T16:13:29,902 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e902e10e767441489a00ea7e24ff1561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255208297 2024-11-10T16:13:29,902 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d181c8842d044b993edbd882ba5427f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:29,903 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5c5476075a483da9e1936fc487669c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:29,921 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:29,922 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:29,922 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/a557338c238a4235af087d8c750de958 is 50, key is test_row_0/B:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:29,922 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/76f83c36ee3d40bbaa3b980d6d12dab8 is 50, key is test_row_0/A:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:29,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741877_1053 (size=12595) 2024-11-10T16:13:29,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741876_1052 (size=12595) 2024-11-10T16:13:29,963 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/a557338c238a4235af087d8c750de958 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a557338c238a4235af087d8c750de958 2024-11-10T16:13:29,968 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/76f83c36ee3d40bbaa3b980d6d12dab8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/76f83c36ee3d40bbaa3b980d6d12dab8 2024-11-10T16:13:29,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T16:13:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,975 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:29,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:29,981 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into a557338c238a4235af087d8c750de958(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:29,981 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:29,981 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255209895; duration=0sec 2024-11-10T16:13:29,986 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:29,986 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:29,987 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:29,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/adf853f75ee84955bf77891758e11017 is 50, key is test_row_0/A:col10/1731255209344/Put/seqid=0 2024-11-10T16:13:29,987 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 76f83c36ee3d40bbaa3b980d6d12dab8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:29,987 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:29,987 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255209894; duration=0sec 2024-11-10T16:13:29,987 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:29,987 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:29,988 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:29,988 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:29,988 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:29,989 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/458b3edf4f1842b485459139ed292abf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=35.9 K 2024-11-10T16:13:29,989 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 458b3edf4f1842b485459139ed292abf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255208153 2024-11-10T16:13:29,990 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6adbb23a171b4044b1f090956e3973d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255208297 2024-11-10T16:13:29,990 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f49de7a597564ea1beda72ff37f19b94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:29,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:30,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741878_1054 (size=12151) 2024-11-10T16:13:30,011 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/adf853f75ee84955bf77891758e11017 2024-11-10T16:13:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255270015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255270015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255270017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,026 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,027 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f4c56694433f47e39f9d055efea48802 is 50, key is test_row_0/C:col10/1731255208676/Put/seqid=0 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/3c981e25be8f425b8603295787a44b5a is 50, key is test_row_0/B:col10/1731255209344/Put/seqid=0 2024-11-10T16:13:30,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255270023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255270029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741879_1055 (size=12595) 2024-11-10T16:13:30,062 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f4c56694433f47e39f9d055efea48802 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4c56694433f47e39f9d055efea48802 2024-11-10T16:13:30,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741880_1056 (size=12151) 2024-11-10T16:13:30,069 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/3c981e25be8f425b8603295787a44b5a 2024-11-10T16:13:30,081 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into f4c56694433f47e39f9d055efea48802(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:30,081 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:30,081 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255209895; duration=0sec 2024-11-10T16:13:30,082 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:30,082 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:30,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/07f593f4ffe74276bb02efa55591caab is 50, key is test_row_0/C:col10/1731255209344/Put/seqid=0 2024-11-10T16:13:30,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741881_1057 (size=12151) 2024-11-10T16:13:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T16:13:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255270134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255270144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255270144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255270343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255270344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255270344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255270354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255270354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,500 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/07f593f4ffe74276bb02efa55591caab 2024-11-10T16:13:30,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/adf853f75ee84955bf77891758e11017 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017 2024-11-10T16:13:30,518 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017, entries=150, sequenceid=209, filesize=11.9 K 2024-11-10T16:13:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/3c981e25be8f425b8603295787a44b5a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a 2024-11-10T16:13:30,531 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-10T16:13:30,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/07f593f4ffe74276bb02efa55591caab as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab 2024-11-10T16:13:30,542 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab, entries=150, sequenceid=209, filesize=11.9 K 2024-11-10T16:13:30,544 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c8fb0b94c7d60b2a8bd105a3337e427b in 569ms, sequenceid=209, compaction requested=false 2024-11-10T16:13:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T16:13:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T16:13:30,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T16:13:30,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0370 sec 2024-11-10T16:13:30,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.0460 sec 2024-11-10T16:13:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T16:13:30,644 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:30,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T16:13:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:30,649 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:30,650 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:30,651 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:30,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:13:30,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:30,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:30,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:30,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:30,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:30,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:30,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/52463c2de8d24000a2472b498d8a7471 is 50, key is test_row_0/A:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:30,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255270676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255270683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255270685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255270686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255270687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741882_1058 (size=14541) 2024-11-10T16:13:30,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/52463c2de8d24000a2472b498d8a7471 2024-11-10T16:13:30,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7b728e60a7574226b5b6815e1caa356e is 50, key is test_row_0/B:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:30,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741883_1059 (size=12151) 2024-11-10T16:13:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:30,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255270793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255270793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255270794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255270804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255270804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:30,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T16:13:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:30,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:30,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:30,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:30,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:30,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T16:13:30,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:30,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:30,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:31,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255271003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255271013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255271014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255271014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255271014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T16:13:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:31,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:31,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7b728e60a7574226b5b6815e1caa356e 2024-11-10T16:13:31,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/abeca55db2414c82b4777037317b6ad4 is 50, key is test_row_0/C:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:31,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741884_1060 (size=12151) 2024-11-10T16:13:31,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/abeca55db2414c82b4777037317b6ad4 2024-11-10T16:13:31,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/52463c2de8d24000a2472b498d8a7471 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471 2024-11-10T16:13:31,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471, entries=200, sequenceid=229, filesize=14.2 K 2024-11-10T16:13:31,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/7b728e60a7574226b5b6815e1caa356e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e 2024-11-10T16:13:31,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e, entries=150, sequenceid=229, filesize=11.9 K 2024-11-10T16:13:31,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/abeca55db2414c82b4777037317b6ad4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4 2024-11-10T16:13:31,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4, entries=150, sequenceid=229, filesize=11.9 K 2024-11-10T16:13:31,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for c8fb0b94c7d60b2a8bd105a3337e427b in 530ms, sequenceid=229, compaction requested=true 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:31,187 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:31,187 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:31,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:31,188 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:31,188 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:31,188 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:31,188 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,188 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:31,189 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/76f83c36ee3d40bbaa3b980d6d12dab8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=38.4 K 2024-11-10T16:13:31,189 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,189 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a557338c238a4235af087d8c750de958, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.0 K 2024-11-10T16:13:31,189 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76f83c36ee3d40bbaa3b980d6d12dab8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:31,189 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a557338c238a4235af087d8c750de958, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:31,190 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf853f75ee84955bf77891758e11017, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731255209344 2024-11-10T16:13:31,190 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c981e25be8f425b8603295787a44b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731255209344 2024-11-10T16:13:31,190 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52463c2de8d24000a2472b498d8a7471, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210012 2024-11-10T16:13:31,190 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b728e60a7574226b5b6815e1caa356e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210014 2024-11-10T16:13:31,205 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:31,207 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cf7fa7b49f3146c98e16067ea8c85006 is 50, key is test_row_0/B:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:31,219 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:31,220 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d2df8fe9c432494abed3fefadb4422b5 is 50, key is test_row_0/A:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:31,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741885_1061 (size=12697) 2024-11-10T16:13:31,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741886_1062 (size=12697) 2024-11-10T16:13:31,248 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d2df8fe9c432494abed3fefadb4422b5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d2df8fe9c432494abed3fefadb4422b5 2024-11-10T16:13:31,260 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into d2df8fe9c432494abed3fefadb4422b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:31,260 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:31,260 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255211187; duration=0sec 2024-11-10T16:13:31,262 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:31,262 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:31,262 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:31,263 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:31,263 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:31,263 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,264 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4c56694433f47e39f9d055efea48802, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.0 K 2024-11-10T16:13:31,264 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4c56694433f47e39f9d055efea48802, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731255208676 2024-11-10T16:13:31,265 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f593f4ffe74276bb02efa55591caab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731255209344 2024-11-10T16:13:31,265 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting abeca55db2414c82b4777037317b6ad4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210014 2024-11-10T16:13:31,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T16:13:31,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,271 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:31,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:31,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:31,279 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:31,280 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/23ee3d5215544a72a350dd301786c9b5 is 50, key is test_row_0/C:col10/1731255210014/Put/seqid=0 2024-11-10T16:13:31,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 is 50, key is test_row_0/A:col10/1731255210684/Put/seqid=0 2024-11-10T16:13:31,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741887_1063 (size=12697) 2024-11-10T16:13:31,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:31,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:31,321 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/23ee3d5215544a72a350dd301786c9b5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/23ee3d5215544a72a350dd301786c9b5 2024-11-10T16:13:31,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741888_1064 (size=12151) 2024-11-10T16:13:31,323 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 2024-11-10T16:13:31,338 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 23ee3d5215544a72a350dd301786c9b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:31,338 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:31,338 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255211187; duration=0sec 2024-11-10T16:13:31,338 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:31,339 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:31,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/00392178b5db44819fb228cc46222faf is 50, key is test_row_0/B:col10/1731255210684/Put/seqid=0 2024-11-10T16:13:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255271344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255271346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255271348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255271357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741889_1065 (size=12151) 2024-11-10T16:13:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255271358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,366 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/00392178b5db44819fb228cc46222faf 2024-11-10T16:13:31,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/301679ad8f634db4bd1939a0f377c554 is 50, key is test_row_0/C:col10/1731255210684/Put/seqid=0 2024-11-10T16:13:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741890_1066 (size=12151) 2024-11-10T16:13:31,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255271464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255271464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255271464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255271473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255271474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,641 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cf7fa7b49f3146c98e16067ea8c85006 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cf7fa7b49f3146c98e16067ea8c85006 2024-11-10T16:13:31,650 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into cf7fa7b49f3146c98e16067ea8c85006(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:31,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:31,650 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255211187; duration=0sec 2024-11-10T16:13:31,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:31,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:31,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255271673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255271674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255271683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255271683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255271684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:31,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:31,796 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/301679ad8f634db4bd1939a0f377c554 2024-11-10T16:13:31,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 2024-11-10T16:13:31,810 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T16:13:31,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/00392178b5db44819fb228cc46222faf as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf 2024-11-10T16:13:31,817 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T16:13:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/301679ad8f634db4bd1939a0f377c554 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554 2024-11-10T16:13:31,823 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T16:13:31,824 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c8fb0b94c7d60b2a8bd105a3337e427b in 554ms, sequenceid=246, compaction requested=false 2024-11-10T16:13:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:31,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T16:13:31,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T16:13:31,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T16:13:31,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1760 sec 2024-11-10T16:13:31,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1860 sec 2024-11-10T16:13:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:31,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:32,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/eed07707a6ab4da4b3f890db16b18d53 is 50, key is test_row_0/A:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741891_1067 (size=12301) 2024-11-10T16:13:32,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/eed07707a6ab4da4b3f890db16b18d53 2024-11-10T16:13:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255272013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255272012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255272019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255272022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255272022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/59738e145e8845adab5665909d24cc4b is 50, key is test_row_0/B:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741892_1068 (size=12301) 2024-11-10T16:13:32,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/59738e145e8845adab5665909d24cc4b 2024-11-10T16:13:32,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6e099f12eddb481eb71c121e040126d8 is 50, key is test_row_0/C:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741893_1069 (size=12301) 2024-11-10T16:13:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255272133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255272133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255272133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255272134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255272134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255272344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255272344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255272344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255272345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255272345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6e099f12eddb481eb71c121e040126d8 2024-11-10T16:13:32,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/eed07707a6ab4da4b3f890db16b18d53 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53 2024-11-10T16:13:32,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T16:13:32,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/59738e145e8845adab5665909d24cc4b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b 2024-11-10T16:13:32,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T16:13:32,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/6e099f12eddb481eb71c121e040126d8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8 2024-11-10T16:13:32,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T16:13:32,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c8fb0b94c7d60b2a8bd105a3337e427b in 497ms, sequenceid=267, compaction requested=true 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:32,493 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:32,493 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:32,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:32,495 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:32,495 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:32,495 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:32,495 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:32,495 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:32,495 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:32,495 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d2df8fe9c432494abed3fefadb4422b5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.3 K 2024-11-10T16:13:32,495 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cf7fa7b49f3146c98e16067ea8c85006, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.3 K 2024-11-10T16:13:32,495 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2df8fe9c432494abed3fefadb4422b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210014 2024-11-10T16:13:32,496 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting cf7fa7b49f3146c98e16067ea8c85006, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210014 2024-11-10T16:13:32,496 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9d0b3b3b4f841fe8de09ae16ae9c0c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731255210674 2024-11-10T16:13:32,496 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 00392178b5db44819fb228cc46222faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731255210674 2024-11-10T16:13:32,496 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting eed07707a6ab4da4b3f890db16b18d53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:32,497 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 59738e145e8845adab5665909d24cc4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:32,511 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:32,512 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:32,512 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/b05e5a1170804bff8e9fa73663a19374 is 50, key is test_row_0/A:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,513 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/f77f43bb70c04d718f6769d2d732878f is 50, key is test_row_0/B:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741894_1070 (size=12949) 2024-11-10T16:13:32,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741895_1071 (size=12949) 2024-11-10T16:13:32,567 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/f77f43bb70c04d718f6769d2d732878f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f77f43bb70c04d718f6769d2d732878f 2024-11-10T16:13:32,575 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into f77f43bb70c04d718f6769d2d732878f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:32,575 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:32,575 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255212493; duration=0sec 2024-11-10T16:13:32,575 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:32,575 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:32,576 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:32,577 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:32,577 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:32,577 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:32,577 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/23ee3d5215544a72a350dd301786c9b5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.3 K 2024-11-10T16:13:32,578 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ee3d5215544a72a350dd301786c9b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731255210014 2024-11-10T16:13:32,579 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 301679ad8f634db4bd1939a0f377c554, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731255210674 2024-11-10T16:13:32,579 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e099f12eddb481eb71c121e040126d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:32,598 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:32,599 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/1f3c6d5443b04a078249e8e8d156dc9a is 50, key is test_row_0/C:col10/1731255211996/Put/seqid=0 2024-11-10T16:13:32,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741896_1072 (size=12949) 2024-11-10T16:13:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:32,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:13:32,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:32,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:32,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:32,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/18ff151f68b24058a7b4c132d85622eb is 50, key is test_row_0/A:col10/1731255212019/Put/seqid=0 2024-11-10T16:13:32,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741897_1073 (size=14741) 2024-11-10T16:13:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255272678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255272680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255272682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255272684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255272684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T16:13:32,794 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:32,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T16:13:32,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255272793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255272793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255272794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255272794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,797 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:32,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T16:13:32,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255272794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:32,799 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:32,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T16:13:32,951 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/b05e5a1170804bff8e9fa73663a19374 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/b05e5a1170804bff8e9fa73663a19374 2024-11-10T16:13:32,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T16:13:32,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:32,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:32,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:32,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:32,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:32,961 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into b05e5a1170804bff8e9fa73663a19374(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:32,961 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:32,961 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255212493; duration=0sec 2024-11-10T16:13:32,962 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:32,962 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255273003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255273003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255273003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255273003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255273004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,017 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/1f3c6d5443b04a078249e8e8d156dc9a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/1f3c6d5443b04a078249e8e8d156dc9a 2024-11-10T16:13:33,024 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 1f3c6d5443b04a078249e8e8d156dc9a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:33,024 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:33,024 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255212493; duration=0sec 2024-11-10T16:13:33,024 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:33,024 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:33,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/18ff151f68b24058a7b4c132d85622eb 2024-11-10T16:13:33,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/386acb8a55c241818d36403a9038b886 is 50, key is test_row_0/B:col10/1731255212019/Put/seqid=0 2024-11-10T16:13:33,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741898_1074 (size=12301) 2024-11-10T16:13:33,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/386acb8a55c241818d36403a9038b886 2024-11-10T16:13:33,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T16:13:33,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d30555385eba461c88cf799285393612 is 50, key is test_row_0/C:col10/1731255212019/Put/seqid=0 2024-11-10T16:13:33,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:33,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:33,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T16:13:33,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741899_1075 (size=12301) 2024-11-10T16:13:33,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d30555385eba461c88cf799285393612 2024-11-10T16:13:33,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/18ff151f68b24058a7b4c132d85622eb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb 2024-11-10T16:13:33,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb, entries=200, sequenceid=283, filesize=14.4 K 2024-11-10T16:13:33,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/386acb8a55c241818d36403a9038b886 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886 2024-11-10T16:13:33,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T16:13:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d30555385eba461c88cf799285393612 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612 2024-11-10T16:13:33,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612, entries=150, sequenceid=283, filesize=12.0 K 2024-11-10T16:13:33,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c8fb0b94c7d60b2a8bd105a3337e427b in 498ms, sequenceid=283, compaction requested=false 2024-11-10T16:13:33,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:33,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T16:13:33,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:33,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:33,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/71041187f64b4a30bb496684d2fb1290 is 50, key is test_row_0/A:col10/1731255212682/Put/seqid=0 2024-11-10T16:13:33,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741900_1076 (size=12301) 2024-11-10T16:13:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:33,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:33,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255273329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255273331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255273332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255273334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255273334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T16:13:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255273443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255273443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255273443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255273444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255273444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255273654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255273654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255273654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255273654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255273654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,682 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/71041187f64b4a30bb496684d2fb1290 2024-11-10T16:13:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/32b369fbd06e41d6ae0d81984c5892c4 is 50, key is test_row_0/B:col10/1731255212682/Put/seqid=0 2024-11-10T16:13:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741901_1077 (size=12301) 2024-11-10T16:13:33,704 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/32b369fbd06e41d6ae0d81984c5892c4 2024-11-10T16:13:33,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e10f3d54bbfb495c9307b409010fe20c is 50, key is test_row_0/C:col10/1731255212682/Put/seqid=0 2024-11-10T16:13:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741902_1078 (size=12301) 2024-11-10T16:13:33,724 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e10f3d54bbfb495c9307b409010fe20c 2024-11-10T16:13:33,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/71041187f64b4a30bb496684d2fb1290 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290 2024-11-10T16:13:33,739 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T16:13:33,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/32b369fbd06e41d6ae0d81984c5892c4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4 2024-11-10T16:13:33,748 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T16:13:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e10f3d54bbfb495c9307b409010fe20c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c 2024-11-10T16:13:33,755 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T16:13:33,757 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for c8fb0b94c7d60b2a8bd105a3337e427b in 493ms, sequenceid=303, compaction requested=true 2024-11-10T16:13:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T16:13:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T16:13:33,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T16:13:33,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 960 msec 2024-11-10T16:13:33,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 969 msec 2024-11-10T16:13:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T16:13:33,933 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:33,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T16:13:33,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:33,938 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:33,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T16:13:33,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:33,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:33,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5316a9838be04c4cbf79bba62b4cd2c7 is 50, key is test_row_0/A:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:33,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741903_1079 (size=12301) 2024-11-10T16:13:33,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5316a9838be04c4cbf79bba62b4cd2c7 2024-11-10T16:13:33,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/b01b0a281f944cbeb91abd3353b49bd6 is 50, key is test_row_0/B:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:33,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255273989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255273995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255273995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:33,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:33,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255273998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255273999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741904_1080 (size=12301) 2024-11-10T16:13:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T16:13:34,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T16:13:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:34,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255274104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255274104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255274104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255274104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255274105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T16:13:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T16:13:34,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255274313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255274314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255274314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255274314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255274314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T16:13:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:34,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/b01b0a281f944cbeb91abd3353b49bd6 2024-11-10T16:13:34,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/2dc0e028449541ee9d6e379968cf4772 is 50, key is test_row_0/C:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:34,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741905_1081 (size=12301) 2024-11-10T16:13:34,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/2dc0e028449541ee9d6e379968cf4772 2024-11-10T16:13:34,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5316a9838be04c4cbf79bba62b4cd2c7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7 2024-11-10T16:13:34,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T16:13:34,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/b01b0a281f944cbeb91abd3353b49bd6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6 2024-11-10T16:13:34,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T16:13:34,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/2dc0e028449541ee9d6e379968cf4772 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772 2024-11-10T16:13:34,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772, entries=150, sequenceid=319, filesize=12.0 K 2024-11-10T16:13:34,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c8fb0b94c7d60b2a8bd105a3337e427b in 490ms, sequenceid=319, compaction requested=true 2024-11-10T16:13:34,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:34,455 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:34,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:34,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:34,457 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:34,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:34,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:34,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:34,458 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:34,461 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:34,461 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:34,461 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,462 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/b05e5a1170804bff8e9fa73663a19374, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=51.1 K 2024-11-10T16:13:34,463 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:34,463 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:34,463 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,463 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f77f43bb70c04d718f6769d2d732878f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=48.7 K 2024-11-10T16:13:34,463 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b05e5a1170804bff8e9fa73663a19374, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:34,464 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f77f43bb70c04d718f6769d2d732878f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:34,464 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ff151f68b24058a7b4c132d85622eb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731255212011 2024-11-10T16:13:34,464 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 386acb8a55c241818d36403a9038b886, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731255212011 2024-11-10T16:13:34,464 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71041187f64b4a30bb496684d2fb1290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731255212665 2024-11-10T16:13:34,465 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 32b369fbd06e41d6ae0d81984c5892c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731255212665 2024-11-10T16:13:34,465 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5316a9838be04c4cbf79bba62b4cd2c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:34,465 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b01b0a281f944cbeb91abd3353b49bd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:34,483 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:34,485 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/65e6ccf20284471aa44fdd77e026f0d0 is 50, key is test_row_0/A:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:34,486 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:34,487 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/709cf08eab084cf98b6f8a8a63f362a9 is 50, key is test_row_0/B:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:34,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741906_1082 (size=13085) 2024-11-10T16:13:34,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741907_1083 (size=13085) 2024-11-10T16:13:34,511 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/709cf08eab084cf98b6f8a8a63f362a9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/709cf08eab084cf98b6f8a8a63f362a9 2024-11-10T16:13:34,519 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 709cf08eab084cf98b6f8a8a63f362a9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:34,519 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:34,519 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=12, startTime=1731255214457; duration=0sec 2024-11-10T16:13:34,519 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:34,519 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:34,520 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:34,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:34,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:34,521 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,522 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/1f3c6d5443b04a078249e8e8d156dc9a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=48.7 K 2024-11-10T16:13:34,522 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3c6d5443b04a078249e8e8d156dc9a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731255211354 2024-11-10T16:13:34,523 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d30555385eba461c88cf799285393612, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731255212011 2024-11-10T16:13:34,523 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e10f3d54bbfb495c9307b409010fe20c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731255212665 2024-11-10T16:13:34,524 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dc0e028449541ee9d6e379968cf4772, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:34,538 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#71 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:34,539 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e67063d5d58c450bb7b5a270c4296425 is 50, key is test_row_0/C:col10/1731255213964/Put/seqid=0 2024-11-10T16:13:34,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741908_1084 (size=13085) 2024-11-10T16:13:34,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T16:13:34,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:34,556 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:34,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:34,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/83a0caa8cd0e499c9d760a015f6b94b8 is 50, key is test_row_0/A:col10/1731255213995/Put/seqid=0 2024-11-10T16:13:34,563 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e67063d5d58c450bb7b5a270c4296425 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e67063d5d58c450bb7b5a270c4296425 2024-11-10T16:13:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T16:13:34,572 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into e67063d5d58c450bb7b5a270c4296425(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:34,572 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:34,572 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=12, startTime=1731255214458; duration=0sec 2024-11-10T16:13:34,572 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:34,572 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:34,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741909_1085 (size=12301) 2024-11-10T16:13:34,586 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/83a0caa8cd0e499c9d760a015f6b94b8 2024-11-10T16:13:34,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/d1c2c0fc4aa84e8b92a405893512ac74 is 50, key is test_row_0/B:col10/1731255213995/Put/seqid=0 2024-11-10T16:13:34,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741910_1086 (size=12301) 2024-11-10T16:13:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:34,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255274655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255274657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255274658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255274660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255274773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255274773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255274773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255274773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255274773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,907 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/65e6ccf20284471aa44fdd77e026f0d0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/65e6ccf20284471aa44fdd77e026f0d0 2024-11-10T16:13:34,913 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 65e6ccf20284471aa44fdd77e026f0d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:34,913 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:34,913 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=12, startTime=1731255214455; duration=0sec 2024-11-10T16:13:34,913 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:34,914 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:34,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255274983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255274983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255274983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255274983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:34,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255274983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:34,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,014 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/d1c2c0fc4aa84e8b92a405893512ac74 2024-11-10T16:13:35,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/33245f3b657b42c4a4de0e7916be0ff5 is 50, key is test_row_0/C:col10/1731255213995/Put/seqid=0 2024-11-10T16:13:35,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741911_1087 (size=12301) 2024-11-10T16:13:35,031 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/33245f3b657b42c4a4de0e7916be0ff5 2024-11-10T16:13:35,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/83a0caa8cd0e499c9d760a015f6b94b8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8 2024-11-10T16:13:35,042 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T16:13:35,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/d1c2c0fc4aa84e8b92a405893512ac74 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74 2024-11-10T16:13:35,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T16:13:35,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/33245f3b657b42c4a4de0e7916be0ff5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5 2024-11-10T16:13:35,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T16:13:35,061 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c8fb0b94c7d60b2a8bd105a3337e427b in 504ms, sequenceid=337, compaction requested=false 2024-11-10T16:13:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T16:13:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T16:13:35,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T16:13:35,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1250 sec 2024-11-10T16:13:35,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1350 sec 2024-11-10T16:13:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T16:13:35,073 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:35,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T16:13:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T16:13:35,076 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:35,077 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:35,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:35,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T16:13:35,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T16:13:35,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:35,231 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:35,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5f1998e96a1448bf99b0c35c0d161ad4 is 50, key is test_row_0/A:col10/1731255214653/Put/seqid=0 2024-11-10T16:13:35,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741912_1088 (size=12301) 2024-11-10T16:13:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:35,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255275358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255275359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255275361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255275362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255275364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T16:13:35,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255275474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255275474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255275474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255275474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255275474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,643 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5f1998e96a1448bf99b0c35c0d161ad4 2024-11-10T16:13:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/fecdc7ab61bf4a87b4db913df3394c43 is 50, key is test_row_0/B:col10/1731255214653/Put/seqid=0 2024-11-10T16:13:35,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741913_1089 (size=12301) 2024-11-10T16:13:35,658 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/fecdc7ab61bf4a87b4db913df3394c43 2024-11-10T16:13:35,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f09d20a7affc4526b4ed41a7beaf5ab3 is 50, key is test_row_0/C:col10/1731255214653/Put/seqid=0 2024-11-10T16:13:35,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741914_1090 (size=12301) 2024-11-10T16:13:35,673 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f09d20a7affc4526b4ed41a7beaf5ab3 2024-11-10T16:13:35,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/5f1998e96a1448bf99b0c35c0d161ad4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4 2024-11-10T16:13:35,684 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4, entries=150, sequenceid=354, filesize=12.0 K 2024-11-10T16:13:35,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/fecdc7ab61bf4a87b4db913df3394c43 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43 2024-11-10T16:13:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255275683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,689 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43, entries=150, sequenceid=354, filesize=12.0 K 2024-11-10T16:13:35,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f09d20a7affc4526b4ed41a7beaf5ab3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3 2024-11-10T16:13:35,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255275693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255275693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255275693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:35,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255275694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:35,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:35,697 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3, entries=150, sequenceid=354, filesize=12.0 K 2024-11-10T16:13:35,698 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c8fb0b94c7d60b2a8bd105a3337e427b in 468ms, sequenceid=354, compaction requested=true 2024-11-10T16:13:35,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:35,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:35,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T16:13:35,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T16:13:35,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T16:13:35,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 623 msec 2024-11-10T16:13:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T16:13:35,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 629 msec 2024-11-10T16:13:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:35,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:36,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a232fc3d83b74b099a95ef1345e317be is 50, key is test_row_0/A:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:36,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741915_1091 (size=14741) 2024-11-10T16:13:36,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255276011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255276012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255276013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255276014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255276019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255276123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255276123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,126 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255276123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255276123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255276133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T16:13:36,214 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:36,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T16:13:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:36,217 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:36,217 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:36,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:36,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255276334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255276334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255276334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255276334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255276343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:36,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a232fc3d83b74b099a95ef1345e317be 2024-11-10T16:13:36,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/a81c918d26ef48429ec4e9c5ef4f37d0 is 50, key is test_row_0/B:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:36,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741916_1092 (size=12301) 2024-11-10T16:13:36,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:36,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:36,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255276643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255276654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255276654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255276655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255276655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:36,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:36,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/a81c918d26ef48429ec4e9c5ef4f37d0 2024-11-10T16:13:36,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:36,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:36,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c7e7bb1423f34191a6e24075407de97a is 50, key is test_row_0/C:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:36,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741917_1093 (size=12301) 2024-11-10T16:13:36,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:36,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:37,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:37,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:37,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:37,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255277163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255277164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:37,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255277164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255277164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255277164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:37,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c7e7bb1423f34191a6e24075407de97a 2024-11-10T16:13:37,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/a232fc3d83b74b099a95ef1345e317be as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be 2024-11-10T16:13:37,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be, entries=200, sequenceid=373, filesize=14.4 K 2024-11-10T16:13:37,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/a81c918d26ef48429ec4e9c5ef4f37d0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0 2024-11-10T16:13:37,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T16:13:37,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/c7e7bb1423f34191a6e24075407de97a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a 2024-11-10T16:13:37,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a, entries=150, sequenceid=373, filesize=12.0 K 2024-11-10T16:13:37,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c8fb0b94c7d60b2a8bd105a3337e427b in 1291ms, sequenceid=373, compaction requested=true 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:37,287 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:37,287 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:37,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:37,289 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:37,289 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52428 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:37,289 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:37,289 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:37,289 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,289 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,289 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/709cf08eab084cf98b6f8a8a63f362a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=48.8 K 2024-11-10T16:13:37,289 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/65e6ccf20284471aa44fdd77e026f0d0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=51.2 K 2024-11-10T16:13:37,290 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 709cf08eab084cf98b6f8a8a63f362a9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:37,290 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e6ccf20284471aa44fdd77e026f0d0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:37,291 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c2c0fc4aa84e8b92a405893512ac74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731255213983 2024-11-10T16:13:37,291 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83a0caa8cd0e499c9d760a015f6b94b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731255213983 2024-11-10T16:13:37,291 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fecdc7ab61bf4a87b4db913df3394c43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731255214642 2024-11-10T16:13:37,292 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f1998e96a1448bf99b0c35c0d161ad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731255214642 2024-11-10T16:13:37,292 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a81c918d26ef48429ec4e9c5ef4f37d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:37,292 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a232fc3d83b74b099a95ef1345e317be, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:37,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:37,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:37,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 is 50, key is test_row_0/A:col10/1731255216011/Put/seqid=0 2024-11-10T16:13:37,314 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#82 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:37,315 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/11e79d0aac354ee8854ba634fe990631 is 50, key is test_row_0/B:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:37,323 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:37,324 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/7eae4e49ce634904b978b011eb19355c is 50, key is test_row_0/A:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:37,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741918_1094 (size=13221) 2024-11-10T16:13:37,350 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/11e79d0aac354ee8854ba634fe990631 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/11e79d0aac354ee8854ba634fe990631 2024-11-10T16:13:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:37,358 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 11e79d0aac354ee8854ba634fe990631(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:37,358 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:37,358 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=12, startTime=1731255217287; duration=0sec 2024-11-10T16:13:37,358 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:37,358 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:37,359 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:37,361 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:37,361 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:37,361 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:37,362 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e67063d5d58c450bb7b5a270c4296425, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=48.8 K 2024-11-10T16:13:37,362 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e67063d5d58c450bb7b5a270c4296425, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731255213330 2024-11-10T16:13:37,363 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 33245f3b657b42c4a4de0e7916be0ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731255213983 2024-11-10T16:13:37,363 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f09d20a7affc4526b4ed41a7beaf5ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731255214642 2024-11-10T16:13:37,364 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e7bb1423f34191a6e24075407de97a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:37,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741919_1095 (size=12301) 2024-11-10T16:13:37,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741920_1096 (size=13221) 2024-11-10T16:13:37,383 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/7eae4e49ce634904b978b011eb19355c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7eae4e49ce634904b978b011eb19355c 2024-11-10T16:13:37,390 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 7eae4e49ce634904b978b011eb19355c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:37,390 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:37,390 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=12, startTime=1731255217287; duration=0sec 2024-11-10T16:13:37,390 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:37,390 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:37,395 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:37,395 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/33bd2edf0f074c09ac278cccf1644e17 is 50, key is test_row_0/C:col10/1731255215358/Put/seqid=0 2024-11-10T16:13:37,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741921_1097 (size=13221) 2024-11-10T16:13:37,415 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/33bd2edf0f074c09ac278cccf1644e17 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33bd2edf0f074c09ac278cccf1644e17 2024-11-10T16:13:37,423 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 33bd2edf0f074c09ac278cccf1644e17(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:37,423 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:37,423 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=12, startTime=1731255217287; duration=0sec 2024-11-10T16:13:37,424 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:37,424 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:37,773 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 2024-11-10T16:13:37,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/49f14637f0d441d396ed4aff8454ee29 is 50, key is test_row_0/B:col10/1731255216011/Put/seqid=0 2024-11-10T16:13:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741922_1098 (size=12301) 2024-11-10T16:13:38,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:38,191 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/49f14637f0d441d396ed4aff8454ee29 2024-11-10T16:13:38,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255278194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255278195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255278196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255278199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,203 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/8662e3f2d8b94d288f293f0cc74ee898 is 50, key is test_row_0/C:col10/1731255216011/Put/seqid=0 2024-11-10T16:13:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255278205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741923_1099 (size=12301) 2024-11-10T16:13:38,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255278314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255278314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255278314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255278314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255278314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255278523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255278523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255278523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255278523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255278523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,615 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/8662e3f2d8b94d288f293f0cc74ee898 2024-11-10T16:13:38,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 2024-11-10T16:13:38,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3, entries=150, sequenceid=386, filesize=12.0 K 2024-11-10T16:13:38,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/49f14637f0d441d396ed4aff8454ee29 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29 2024-11-10T16:13:38,633 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29, entries=150, sequenceid=386, filesize=12.0 K 2024-11-10T16:13:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/8662e3f2d8b94d288f293f0cc74ee898 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898 2024-11-10T16:13:38,640 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898, entries=150, sequenceid=386, filesize=12.0 K 2024-11-10T16:13:38,642 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c8fb0b94c7d60b2a8bd105a3337e427b in 1347ms, sequenceid=386, compaction requested=false 2024-11-10T16:13:38,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:38,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:38,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T16:13:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T16:13:38,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T16:13:38,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4270 sec 2024-11-10T16:13:38,648 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.4330 sec 2024-11-10T16:13:38,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:38,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:38,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:38,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d120849ae71e463591e42ae86a7746b1 is 50, key is test_row_0/A:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:38,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741924_1100 (size=14741) 2024-11-10T16:13:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255278844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255278846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255278895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255278895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:38,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255278898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:38,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255279003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255279003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255279003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255279003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255279013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255279213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255279213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255279213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255279214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255279223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d120849ae71e463591e42ae86a7746b1 2024-11-10T16:13:39,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/149136e455174de5b115a8ed1519c712 is 50, key is test_row_0/B:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:39,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741925_1101 (size=12301) 2024-11-10T16:13:39,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/149136e455174de5b115a8ed1519c712 2024-11-10T16:13:39,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d2cf08a4d3184764a861d7c66c69e3b0 is 50, key is test_row_0/C:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:39,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741926_1102 (size=12301) 2024-11-10T16:13:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255279523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255279523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255279523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255279523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255279534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:39,723 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d2cf08a4d3184764a861d7c66c69e3b0 2024-11-10T16:13:39,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/d120849ae71e463591e42ae86a7746b1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1 2024-11-10T16:13:39,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1, entries=200, sequenceid=409, filesize=14.4 K 2024-11-10T16:13:39,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/149136e455174de5b115a8ed1519c712 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712 2024-11-10T16:13:39,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T16:13:39,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/d2cf08a4d3184764a861d7c66c69e3b0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0 2024-11-10T16:13:39,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T16:13:39,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c8fb0b94c7d60b2a8bd105a3337e427b in 925ms, sequenceid=409, compaction requested=true 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:39,760 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:39,760 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:39,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:39,763 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:39,763 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:39,763 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:39,763 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/11e79d0aac354ee8854ba634fe990631, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.9 K 2024-11-10T16:13:39,763 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:39,763 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:39,763 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:39,763 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7eae4e49ce634904b978b011eb19355c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=39.3 K 2024-11-10T16:13:39,764 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e79d0aac354ee8854ba634fe990631, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:39,764 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eae4e49ce634904b978b011eb19355c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:39,765 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f14637f0d441d396ed4aff8454ee29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731255216006 2024-11-10T16:13:39,765 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6a24f887e7c4bacbddeec5e1f7ea3e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731255216006 2024-11-10T16:13:39,765 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 149136e455174de5b115a8ed1519c712, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:39,766 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d120849ae71e463591e42ae86a7746b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:39,788 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:39,789 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/81552e45ea2042a288ca21f89fb307f3 is 50, key is test_row_0/A:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:39,790 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:39,791 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/5ceb175b34224dbc8212e4f8f90e0b96 is 50, key is test_row_0/B:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741927_1103 (size=13323) 2024-11-10T16:13:39,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741928_1104 (size=13323) 2024-11-10T16:13:39,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/5ceb175b34224dbc8212e4f8f90e0b96 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5ceb175b34224dbc8212e4f8f90e0b96 2024-11-10T16:13:39,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 5ceb175b34224dbc8212e4f8f90e0b96(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:39,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:39,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=13, startTime=1731255219760; duration=0sec 2024-11-10T16:13:39,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:39,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:39,838 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:39,840 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:39,840 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:39,840 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:39,840 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33bd2edf0f074c09ac278cccf1644e17, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=36.9 K 2024-11-10T16:13:39,841 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 33bd2edf0f074c09ac278cccf1644e17, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731255215358 2024-11-10T16:13:39,841 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8662e3f2d8b94d288f293f0cc74ee898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731255216006 2024-11-10T16:13:39,842 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cf08a4d3184764a861d7c66c69e3b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:39,853 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#92 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:39,854 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/64a42213846249128e541c64461564db is 50, key is test_row_0/C:col10/1731255218194/Put/seqid=0 2024-11-10T16:13:39,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741929_1105 (size=13323) 2024-11-10T16:13:39,869 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/64a42213846249128e541c64461564db as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/64a42213846249128e541c64461564db 2024-11-10T16:13:39,876 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 64a42213846249128e541c64461564db(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:39,876 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:39,876 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=13, startTime=1731255219760; duration=0sec 2024-11-10T16:13:39,877 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:39,877 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:40,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:40,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:40,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f5623ec8217141b492c69b90f41533f7 is 50, key is test_row_0/A:col10/1731255218847/Put/seqid=0 2024-11-10T16:13:40,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255280060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255280060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255280061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255280062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255280068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741930_1106 (size=9857) 2024-11-10T16:13:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255280174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255280174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255280174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255280174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255280183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,221 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/81552e45ea2042a288ca21f89fb307f3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81552e45ea2042a288ca21f89fb307f3 2024-11-10T16:13:40,228 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 81552e45ea2042a288ca21f89fb307f3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:40,228 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:40,228 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=13, startTime=1731255219760; duration=0sec 2024-11-10T16:13:40,228 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:40,228 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T16:13:40,385 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:40,386 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-10T16:13:40,389 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:40,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255280384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,390 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255280384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:40,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255280385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255280385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255280393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f5623ec8217141b492c69b90f41533f7 2024-11-10T16:13:40,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/4b2269ebc98041b9814e299583a8e833 is 50, key is test_row_0/B:col10/1731255218847/Put/seqid=0 2024-11-10T16:13:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:40,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741931_1107 (size=9857) 2024-11-10T16:13:40,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:40,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255280693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255280703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255280703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255280703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255280704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:40,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:40,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:40,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/4b2269ebc98041b9814e299583a8e833 2024-11-10T16:13:40,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/0364bf27c2844cedaefa1b443a997d20 is 50, key is test_row_0/C:col10/1731255218847/Put/seqid=0 2024-11-10T16:13:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741932_1108 (size=9857) 2024-11-10T16:13:41,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:41,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:41,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255281203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:41,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:41,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:41,223 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:41,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:41,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/0364bf27c2844cedaefa1b443a997d20 2024-11-10T16:13:41,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/f5623ec8217141b492c69b90f41533f7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7 2024-11-10T16:13:41,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7, entries=100, sequenceid=424, filesize=9.6 K 2024-11-10T16:13:41,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/4b2269ebc98041b9814e299583a8e833 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833 2024-11-10T16:13:41,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833, entries=100, sequenceid=424, filesize=9.6 K 2024-11-10T16:13:41,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/0364bf27c2844cedaefa1b443a997d20 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20 2024-11-10T16:13:41,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20, entries=100, sequenceid=424, filesize=9.6 K 2024-11-10T16:13:41,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c8fb0b94c7d60b2a8bd105a3337e427b in 1306ms, sequenceid=424, compaction requested=false 2024-11-10T16:13:41,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:41,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-10T16:13:41,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:41,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:41,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:41,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/541b739742394c8b8b904413e1ef19fc is 50, key is test_row_0/A:col10/1731255220066/Put/seqid=0 2024-11-10T16:13:41,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741933_1109 (size=12301) 2024-11-10T16:13:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:41,879 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/541b739742394c8b8b904413e1ef19fc 2024-11-10T16:13:41,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/42a5af7fee424996b3420e6046a25d3e is 50, key is test_row_0/B:col10/1731255220066/Put/seqid=0 2024-11-10T16:13:41,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741934_1110 (size=12301) 2024-11-10T16:13:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:42,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255282239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255282239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255282240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255282242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255282243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,294 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/42a5af7fee424996b3420e6046a25d3e 2024-11-10T16:13:42,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/34fdb9bbbcf24275ad555af8a79770ce is 50, key is test_row_0/C:col10/1731255220066/Put/seqid=0 2024-11-10T16:13:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741935_1111 (size=12301) 2024-11-10T16:13:42,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255282353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255282353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255282353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255282353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255282353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255282563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255282563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255282563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255282564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255282564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,708 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/34fdb9bbbcf24275ad555af8a79770ce 2024-11-10T16:13:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/541b739742394c8b8b904413e1ef19fc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc 2024-11-10T16:13:42,720 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc, entries=150, sequenceid=444, filesize=12.0 K 2024-11-10T16:13:42,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/42a5af7fee424996b3420e6046a25d3e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e 2024-11-10T16:13:42,727 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e, entries=150, sequenceid=444, filesize=12.0 K 2024-11-10T16:13:42,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/34fdb9bbbcf24275ad555af8a79770ce as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce 2024-11-10T16:13:42,733 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce, entries=150, sequenceid=444, filesize=12.0 K 2024-11-10T16:13:42,734 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c8fb0b94c7d60b2a8bd105a3337e427b in 1268ms, sequenceid=444, compaction requested=true 2024-11-10T16:13:42,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:42,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:42,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-10T16:13:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-10T16:13:42,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T16:13:42,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3460 sec 2024-11-10T16:13:42,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3550 sec 2024-11-10T16:13:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:42,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:42,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:42,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/3dd117a7a0c04b7c8949c6d87b1178ab is 50, key is test_row_0/A:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:42,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741936_1112 (size=17181) 2024-11-10T16:13:42,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255282892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255282893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255282894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255282898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255282900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255283003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255283003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255283003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255283003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255283013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255283213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255283213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255283213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255283214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255283224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/3dd117a7a0c04b7c8949c6d87b1178ab 2024-11-10T16:13:43,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/0e75c47852824d979e4b96fd9a9ed1e7 is 50, key is test_row_0/B:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741937_1113 (size=12301) 2024-11-10T16:13:43,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/0e75c47852824d979e4b96fd9a9ed1e7 2024-11-10T16:13:43,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/90b3aa461dd341ffb31926ff4027228b is 50, key is test_row_0/C:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:43,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741938_1114 (size=12301) 2024-11-10T16:13:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255283523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255283523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255283523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255283523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:43,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255283533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:43,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:43,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/90b3aa461dd341ffb31926ff4027228b 2024-11-10T16:13:43,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/3dd117a7a0c04b7c8949c6d87b1178ab as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab 2024-11-10T16:13:43,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab, entries=250, sequenceid=459, filesize=16.8 K 2024-11-10T16:13:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/0e75c47852824d979e4b96fd9a9ed1e7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7 2024-11-10T16:13:43,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T16:13:43,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/90b3aa461dd341ffb31926ff4027228b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b 2024-11-10T16:13:43,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T16:13:43,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c8fb0b94c7d60b2a8bd105a3337e427b in 863ms, sequenceid=459, compaction requested=true 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:43,741 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8fb0b94c7d60b2a8bd105a3337e427b:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:43,741 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:43,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52662 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/A is initiating minor compaction (all files) 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/B is initiating minor compaction (all files) 2024-11-10T16:13:43,743 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/B in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:43,743 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/A in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:43,743 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5ceb175b34224dbc8212e4f8f90e0b96, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=46.7 K 2024-11-10T16:13:43,743 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81552e45ea2042a288ca21f89fb307f3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=51.4 K 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ceb175b34224dbc8212e4f8f90e0b96, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:43,743 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81552e45ea2042a288ca21f89fb307f3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:43,744 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b2269ebc98041b9814e299583a8e833, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255218847 2024-11-10T16:13:43,744 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5623ec8217141b492c69b90f41533f7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255218847 2024-11-10T16:13:43,744 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a5af7fee424996b3420e6046a25d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731255220053 2024-11-10T16:13:43,744 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 541b739742394c8b8b904413e1ef19fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731255220053 2024-11-10T16:13:43,745 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e75c47852824d979e4b96fd9a9ed1e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731255222225 2024-11-10T16:13:43,745 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd117a7a0c04b7c8949c6d87b1178ab, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731255222225 2024-11-10T16:13:43,758 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:43,758 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:43,758 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/7ca36a4224ca4038bd478533e8934b6d is 50, key is test_row_0/A:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:43,759 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/20a73feefdec485aaee7a87115004ca4 is 50, key is test_row_0/B:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:43,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741939_1115 (size=13459) 2024-11-10T16:13:43,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741940_1116 (size=13459) 2024-11-10T16:13:43,774 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/20a73feefdec485aaee7a87115004ca4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/20a73feefdec485aaee7a87115004ca4 2024-11-10T16:13:43,780 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/B of c8fb0b94c7d60b2a8bd105a3337e427b into 20a73feefdec485aaee7a87115004ca4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:43,780 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:43,780 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/B, priority=12, startTime=1731255223741; duration=0sec 2024-11-10T16:13:43,781 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:43,781 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:B 2024-11-10T16:13:43,781 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:43,783 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:43,783 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): c8fb0b94c7d60b2a8bd105a3337e427b/C is initiating minor compaction (all files) 2024-11-10T16:13:43,783 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8fb0b94c7d60b2a8bd105a3337e427b/C in TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:43,783 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/64a42213846249128e541c64461564db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp, totalSize=46.7 K 2024-11-10T16:13:43,784 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a42213846249128e541c64461564db, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255218190 2024-11-10T16:13:43,784 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0364bf27c2844cedaefa1b443a997d20, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255218847 2024-11-10T16:13:43,785 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 34fdb9bbbcf24275ad555af8a79770ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1731255220053 2024-11-10T16:13:43,785 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b3aa461dd341ffb31926ff4027228b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=459, earliestPutTs=1731255222225 2024-11-10T16:13:43,798 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8fb0b94c7d60b2a8bd105a3337e427b#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:43,799 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/4658cd76c4a94fe9a024f92968789863 is 50, key is test_row_0/C:col10/1731255222877/Put/seqid=0 2024-11-10T16:13:43,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741941_1117 (size=13459) 2024-11-10T16:13:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:44,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:44,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:44,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:44,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c2f2be8864894514ae6b7d36bf955afa is 50, key is test_row_0/A:col10/1731255224035/Put/seqid=0 2024-11-10T16:13:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741942_1118 (size=17181) 2024-11-10T16:13:44,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c2f2be8864894514ae6b7d36bf955afa 2024-11-10T16:13:44,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/329b3a26a29b4d998a16661238248ad0 is 50, key is test_row_0/B:col10/1731255224035/Put/seqid=0 2024-11-10T16:13:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741943_1119 (size=12301) 2024-11-10T16:13:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255284053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,066 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255284063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255284065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255284065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255284066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,171 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/7ca36a4224ca4038bd478533e8934b6d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7ca36a4224ca4038bd478533e8934b6d 2024-11-10T16:13:44,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255284173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255284174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,179 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/A of c8fb0b94c7d60b2a8bd105a3337e427b into 7ca36a4224ca4038bd478533e8934b6d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:44,179 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:44,179 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/A, priority=12, startTime=1731255223741; duration=0sec 2024-11-10T16:13:44,179 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:44,179 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:A 2024-11-10T16:13:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255284183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,211 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/4658cd76c4a94fe9a024f92968789863 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/4658cd76c4a94fe9a024f92968789863 2024-11-10T16:13:44,217 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8fb0b94c7d60b2a8bd105a3337e427b/C of c8fb0b94c7d60b2a8bd105a3337e427b into 4658cd76c4a94fe9a024f92968789863(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:44,217 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:44,217 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., storeName=c8fb0b94c7d60b2a8bd105a3337e427b/C, priority=12, startTime=1731255223741; duration=0sec 2024-11-10T16:13:44,217 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:44,217 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8fb0b94c7d60b2a8bd105a3337e427b:C 2024-11-10T16:13:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255284383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255284383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,385 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255284383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255284383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255284393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/329b3a26a29b4d998a16661238248ad0 2024-11-10T16:13:44,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e796cb48e277440d9bbcc2d2ffca9d01 is 50, key is test_row_0/C:col10/1731255224035/Put/seqid=0 2024-11-10T16:13:44,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741944_1120 (size=12301) 2024-11-10T16:13:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T16:13:44,554 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:44,554 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-10T16:13:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T16:13:44,556 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:44,557 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:44,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T16:13:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38394 deadline: 1731255284693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38452 deadline: 1731255284693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38416 deadline: 1731255284703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38426 deadline: 1731255284703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:44,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38384 deadline: 1731255284703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:44,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:13:44,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T16:13:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:44,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T16:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:44,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:44,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T16:13:44,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e796cb48e277440d9bbcc2d2ffca9d01 2024-11-10T16:13:44,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/c2f2be8864894514ae6b7d36bf955afa as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c2f2be8864894514ae6b7d36bf955afa 2024-11-10T16:13:44,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c2f2be8864894514ae6b7d36bf955afa, entries=250, sequenceid=479, filesize=16.8 K 2024-11-10T16:13:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/329b3a26a29b4d998a16661238248ad0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/329b3a26a29b4d998a16661238248ad0 2024-11-10T16:13:44,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/329b3a26a29b4d998a16661238248ad0, entries=150, sequenceid=479, filesize=12.0 K 2024-11-10T16:13:44,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e796cb48e277440d9bbcc2d2ffca9d01 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e796cb48e277440d9bbcc2d2ffca9d01 2024-11-10T16:13:44,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e796cb48e277440d9bbcc2d2ffca9d01, entries=150, sequenceid=479, filesize=12.0 K 2024-11-10T16:13:44,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c8fb0b94c7d60b2a8bd105a3337e427b in 885ms, sequenceid=479, compaction requested=false 2024-11-10T16:13:44,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:45,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-10T16:13:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:45,017 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:45,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e5877d7185d64b0685da74e860ba62d3 is 50, key is test_row_0/A:col10/1731255224063/Put/seqid=0 2024-11-10T16:13:45,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741945_1121 (size=12301) 2024-11-10T16:13:45,029 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e5877d7185d64b0685da74e860ba62d3 2024-11-10T16:13:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/60f53a5a54bc48218e144342cd76958c is 50, key is test_row_0/B:col10/1731255224063/Put/seqid=0 2024-11-10T16:13:45,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741946_1122 (size=12301) 2024-11-10T16:13:45,045 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/60f53a5a54bc48218e144342cd76958c 2024-11-10T16:13:45,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f4292fab5aba49caa82c80c59ac53a44 is 50, key is test_row_0/C:col10/1731255224063/Put/seqid=0 2024-11-10T16:13:45,058 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T16:13:45,058 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,058 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T16:13:45,058 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,060 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T16:13:45,060 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,062 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T16:13:45,062 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,063 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,063 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,063 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,063 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,064 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,064 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,064 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,064 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741947_1123 (size=12301) 2024-11-10T16:13:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T16:13:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. as already flushing 2024-11-10T16:13:45,205 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T16:13:45,205 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T16:13:45,205 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,205 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,205 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,205 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,205 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,205 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,205 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,215 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T16:13:45,215 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,215 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,215 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T16:13:45,215 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,215 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,215 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,215 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,224 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T16:13:45,224 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:13:45,224 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,224 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,468 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f4292fab5aba49caa82c80c59ac53a44 2024-11-10T16:13:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/e5877d7185d64b0685da74e860ba62d3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e5877d7185d64b0685da74e860ba62d3 2024-11-10T16:13:45,478 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e5877d7185d64b0685da74e860ba62d3, entries=150, sequenceid=495, filesize=12.0 K 2024-11-10T16:13:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/60f53a5a54bc48218e144342cd76958c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/60f53a5a54bc48218e144342cd76958c 2024-11-10T16:13:45,482 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/60f53a5a54bc48218e144342cd76958c, entries=150, sequenceid=495, filesize=12.0 K 2024-11-10T16:13:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/f4292fab5aba49caa82c80c59ac53a44 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4292fab5aba49caa82c80c59ac53a44 2024-11-10T16:13:45,487 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4292fab5aba49caa82c80c59ac53a44, entries=150, sequenceid=495, filesize=12.0 K 2024-11-10T16:13:45,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for c8fb0b94c7d60b2a8bd105a3337e427b in 471ms, sequenceid=495, compaction requested=true 2024-11-10T16:13:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for c8fb0b94c7d60b2a8bd105a3337e427b: 2024-11-10T16:13:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:45,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-10T16:13:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-10T16:13:45,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T16:13:45,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 932 msec 2024-11-10T16:13:45,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 937 msec 2024-11-10T16:13:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-10T16:13:45,693 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:45,693 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2793 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2781 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1272 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3816 rows 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1266 2024-11-10T16:13:45,694 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3794 rows 2024-11-10T16:13:45,694 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:13:45,694 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:13:45,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:45,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:13:45,699 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:13:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:45,706 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255225705"}]},"ts":"1731255225705"} 2024-11-10T16:13:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:45,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:13:45,707 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:13:45,709 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:13:45,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, UNASSIGN}] 2024-11-10T16:13:45,714 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, UNASSIGN 2024-11-10T16:13:45,715 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c8fb0b94c7d60b2a8bd105a3337e427b, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:45,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, UNASSIGN because future has completed 2024-11-10T16:13:45,717 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:13:45,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:45,873 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:45,874 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:13:45,874 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing c8fb0b94c7d60b2a8bd105a3337e427b, disabling compactions & flushes 2024-11-10T16:13:45,874 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:45,874 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:45,874 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. after waiting 0 ms 2024-11-10T16:13:45,874 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:45,874 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing c8fb0b94c7d60b2a8bd105a3337e427b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=A 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=B 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c8fb0b94c7d60b2a8bd105a3337e427b, store=C 2024-11-10T16:13:45,875 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:45,879 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/94b481189892452b9654394cffe7b710 is 50, key is test_row_1/A:col10/1731255225223/Put/seqid=0 2024-11-10T16:13:45,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741948_1124 (size=9857) 2024-11-10T16:13:46,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:46,284 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/94b481189892452b9654394cffe7b710 2024-11-10T16:13:46,292 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cbd984de27b34b9d9f81e38c7bbd86bb is 50, key is test_row_1/B:col10/1731255225223/Put/seqid=0 2024-11-10T16:13:46,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741949_1125 (size=9857) 2024-11-10T16:13:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:46,696 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cbd984de27b34b9d9f81e38c7bbd86bb 2024-11-10T16:13:46,703 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e389ffdcf76243b4970da76b161bf456 is 50, key is test_row_1/C:col10/1731255225223/Put/seqid=0 2024-11-10T16:13:46,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741950_1126 (size=9857) 2024-11-10T16:13:46,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:47,108 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e389ffdcf76243b4970da76b161bf456 2024-11-10T16:13:47,113 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/A/94b481189892452b9654394cffe7b710 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/94b481189892452b9654394cffe7b710 2024-11-10T16:13:47,117 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/94b481189892452b9654394cffe7b710, entries=100, sequenceid=503, filesize=9.6 K 2024-11-10T16:13:47,118 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/B/cbd984de27b34b9d9f81e38c7bbd86bb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cbd984de27b34b9d9f81e38c7bbd86bb 2024-11-10T16:13:47,122 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cbd984de27b34b9d9f81e38c7bbd86bb, entries=100, sequenceid=503, filesize=9.6 K 2024-11-10T16:13:47,123 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/.tmp/C/e389ffdcf76243b4970da76b161bf456 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e389ffdcf76243b4970da76b161bf456 2024-11-10T16:13:47,127 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e389ffdcf76243b4970da76b161bf456, entries=100, sequenceid=503, filesize=9.6 K 2024-11-10T16:13:47,128 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c8fb0b94c7d60b2a8bd105a3337e427b in 1254ms, sequenceid=503, compaction requested=true 2024-11-10T16:13:47,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c77831f186c54b1daad860d1ee6a9a5e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2e44ad63c25a4f32a3927fd821c2748d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/fa9efac19aea4bafbc62d298f0545279, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/76f83c36ee3d40bbaa3b980d6d12dab8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d2df8fe9c432494abed3fefadb4422b5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/b05e5a1170804bff8e9fa73663a19374, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/65e6ccf20284471aa44fdd77e026f0d0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7eae4e49ce634904b978b011eb19355c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81552e45ea2042a288ca21f89fb307f3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab] to archive 2024-11-10T16:13:47,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:13:47,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/48a2ff327be7481db556054c0957113f 2024-11-10T16:13:47,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81e4f9e0a6a343deb2332dc37379de4e 2024-11-10T16:13:47,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/ed2b1607b3ba4a31983fbde3918f4c8d 2024-11-10T16:13:47,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c77831f186c54b1daad860d1ee6a9a5e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c77831f186c54b1daad860d1ee6a9a5e 2024-11-10T16:13:47,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f0cae60e095e4eb989cd655c1997888a 2024-11-10T16:13:47,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c4c14f6531ed471ebb4e3ae45c7e6b15 2024-11-10T16:13:47,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2e44ad63c25a4f32a3927fd821c2748d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2e44ad63c25a4f32a3927fd821c2748d 2024-11-10T16:13:47,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/dea8dbea4f394db38219f64a9ec841f4 2024-11-10T16:13:47,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2dacc0c8a2504e6f86df745112427110 2024-11-10T16:13:47,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a03c1a47a69144ad86c8f01068d5ce46 2024-11-10T16:13:47,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/fa9efac19aea4bafbc62d298f0545279 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/fa9efac19aea4bafbc62d298f0545279 2024-11-10T16:13:47,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d6d1ce4141a34287a9b1e9c75120e536 2024-11-10T16:13:47,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/2d181c8842d044b993edbd882ba5427f 2024-11-10T16:13:47,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/76f83c36ee3d40bbaa3b980d6d12dab8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/76f83c36ee3d40bbaa3b980d6d12dab8 2024-11-10T16:13:47,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/adf853f75ee84955bf77891758e11017 2024-11-10T16:13:47,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/52463c2de8d24000a2472b498d8a7471 2024-11-10T16:13:47,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d2df8fe9c432494abed3fefadb4422b5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d2df8fe9c432494abed3fefadb4422b5 2024-11-10T16:13:47,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c9d0b3b3b4f841fe8de09ae16ae9c0c1 2024-11-10T16:13:47,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/b05e5a1170804bff8e9fa73663a19374 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/b05e5a1170804bff8e9fa73663a19374 2024-11-10T16:13:47,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/eed07707a6ab4da4b3f890db16b18d53 2024-11-10T16:13:47,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/18ff151f68b24058a7b4c132d85622eb 2024-11-10T16:13:47,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/71041187f64b4a30bb496684d2fb1290 2024-11-10T16:13:47,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/65e6ccf20284471aa44fdd77e026f0d0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/65e6ccf20284471aa44fdd77e026f0d0 2024-11-10T16:13:47,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5316a9838be04c4cbf79bba62b4cd2c7 2024-11-10T16:13:47,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/83a0caa8cd0e499c9d760a015f6b94b8 2024-11-10T16:13:47,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/5f1998e96a1448bf99b0c35c0d161ad4 2024-11-10T16:13:47,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/a232fc3d83b74b099a95ef1345e317be 2024-11-10T16:13:47,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7eae4e49ce634904b978b011eb19355c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7eae4e49ce634904b978b011eb19355c 2024-11-10T16:13:47,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e6a24f887e7c4bacbddeec5e1f7ea3e3 2024-11-10T16:13:47,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/d120849ae71e463591e42ae86a7746b1 2024-11-10T16:13:47,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81552e45ea2042a288ca21f89fb307f3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/81552e45ea2042a288ca21f89fb307f3 2024-11-10T16:13:47,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/f5623ec8217141b492c69b90f41533f7 2024-11-10T16:13:47,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/541b739742394c8b8b904413e1ef19fc 2024-11-10T16:13:47,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/3dd117a7a0c04b7c8949c6d87b1178ab 2024-11-10T16:13:47,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/84306a79008542beba8e490d80eb6404, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e427fae697c84dcd8d3db17f5cabffa4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/9f122800fd2141619db55416d14da53c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a557338c238a4235af087d8c750de958, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cf7fa7b49f3146c98e16067ea8c85006, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f77f43bb70c04d718f6769d2d732878f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/709cf08eab084cf98b6f8a8a63f362a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/11e79d0aac354ee8854ba634fe990631, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5ceb175b34224dbc8212e4f8f90e0b96, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7] to archive 2024-11-10T16:13:47,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:13:47,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/538cefe891534acca135215665635d9a 2024-11-10T16:13:47,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cec06e5889f54760b113dafcc1b93941 2024-11-10T16:13:47,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/1897ebdd37364289b5ae43dbad910b1d 2024-11-10T16:13:47,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/84306a79008542beba8e490d80eb6404 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/84306a79008542beba8e490d80eb6404 2024-11-10T16:13:47,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f49b02b560d745b2a623baeb69250ba2 2024-11-10T16:13:47,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/8f80a98d6b034f52b88d77a2aeeaa247 2024-11-10T16:13:47,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e427fae697c84dcd8d3db17f5cabffa4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e427fae697c84dcd8d3db17f5cabffa4 2024-11-10T16:13:47,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5a1a2d5ce66146629e8881ea787431fc 2024-11-10T16:13:47,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/ee92a610529f4e4c9075f07c692b1af9 2024-11-10T16:13:47,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/9f122800fd2141619db55416d14da53c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/9f122800fd2141619db55416d14da53c 2024-11-10T16:13:47,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/362cfc33fb474ee6a1b31ec1e738d48e 2024-11-10T16:13:47,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/e902e10e767441489a00ea7e24ff1561 2024-11-10T16:13:47,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a557338c238a4235af087d8c750de958 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a557338c238a4235af087d8c750de958 2024-11-10T16:13:47,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7a5c5476075a483da9e1936fc487669c 2024-11-10T16:13:47,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/3c981e25be8f425b8603295787a44b5a 2024-11-10T16:13:47,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cf7fa7b49f3146c98e16067ea8c85006 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cf7fa7b49f3146c98e16067ea8c85006 2024-11-10T16:13:47,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/7b728e60a7574226b5b6815e1caa356e 2024-11-10T16:13:47,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/00392178b5db44819fb228cc46222faf 2024-11-10T16:13:47,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f77f43bb70c04d718f6769d2d732878f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/f77f43bb70c04d718f6769d2d732878f 2024-11-10T16:13:47,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/59738e145e8845adab5665909d24cc4b 2024-11-10T16:13:47,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/386acb8a55c241818d36403a9038b886 2024-11-10T16:13:47,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/32b369fbd06e41d6ae0d81984c5892c4 2024-11-10T16:13:47,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/709cf08eab084cf98b6f8a8a63f362a9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/709cf08eab084cf98b6f8a8a63f362a9 2024-11-10T16:13:47,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/b01b0a281f944cbeb91abd3353b49bd6 2024-11-10T16:13:47,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/d1c2c0fc4aa84e8b92a405893512ac74 2024-11-10T16:13:47,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/fecdc7ab61bf4a87b4db913df3394c43 2024-11-10T16:13:47,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/11e79d0aac354ee8854ba634fe990631 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/11e79d0aac354ee8854ba634fe990631 2024-11-10T16:13:47,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/a81c918d26ef48429ec4e9c5ef4f37d0 2024-11-10T16:13:47,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/49f14637f0d441d396ed4aff8454ee29 2024-11-10T16:13:47,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5ceb175b34224dbc8212e4f8f90e0b96 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/5ceb175b34224dbc8212e4f8f90e0b96 2024-11-10T16:13:47,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/149136e455174de5b115a8ed1519c712 2024-11-10T16:13:47,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/4b2269ebc98041b9814e299583a8e833 2024-11-10T16:13:47,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/42a5af7fee424996b3420e6046a25d3e 2024-11-10T16:13:47,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/0e75c47852824d979e4b96fd9a9ed1e7 2024-11-10T16:13:47,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6a6a190cc1f24cdcad5bccaca7b38b92, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9b6ff284e7fa494a980d216066793e15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/458b3edf4f1842b485459139ed292abf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4c56694433f47e39f9d055efea48802, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/23ee3d5215544a72a350dd301786c9b5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/1f3c6d5443b04a078249e8e8d156dc9a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e67063d5d58c450bb7b5a270c4296425, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33bd2edf0f074c09ac278cccf1644e17, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/64a42213846249128e541c64461564db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b] to archive 2024-11-10T16:13:47,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:13:47,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9d4a19ec55674956b88791507a411a51 2024-11-10T16:13:47,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/3ae05e33fea24195b8bb23bf9a725b36 2024-11-10T16:13:47,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/5ee0026a67564a90be0e6327033f427a 2024-11-10T16:13:47,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6a6a190cc1f24cdcad5bccaca7b38b92 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6a6a190cc1f24cdcad5bccaca7b38b92 2024-11-10T16:13:47,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c815b9136b4940fbadfa5a556547306d 2024-11-10T16:13:47,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/521d0fa03ac244cc8433ae293af981e4 2024-11-10T16:13:47,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9b6ff284e7fa494a980d216066793e15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/9b6ff284e7fa494a980d216066793e15 2024-11-10T16:13:47,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/758950887f16461faa76242f90636582 2024-11-10T16:13:47,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/26016c9fa8e74296a6e97dcc5616d925 2024-11-10T16:13:47,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/458b3edf4f1842b485459139ed292abf to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/458b3edf4f1842b485459139ed292abf 2024-11-10T16:13:47,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c0f23d655c1a45df950358bd553e8238 2024-11-10T16:13:47,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6adbb23a171b4044b1f090956e3973d7 2024-11-10T16:13:47,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4c56694433f47e39f9d055efea48802 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4c56694433f47e39f9d055efea48802 2024-11-10T16:13:47,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f49de7a597564ea1beda72ff37f19b94 2024-11-10T16:13:47,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/07f593f4ffe74276bb02efa55591caab 2024-11-10T16:13:47,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/23ee3d5215544a72a350dd301786c9b5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/23ee3d5215544a72a350dd301786c9b5 2024-11-10T16:13:47,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/abeca55db2414c82b4777037317b6ad4 2024-11-10T16:13:47,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/301679ad8f634db4bd1939a0f377c554 2024-11-10T16:13:47,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/1f3c6d5443b04a078249e8e8d156dc9a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/1f3c6d5443b04a078249e8e8d156dc9a 2024-11-10T16:13:47,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/6e099f12eddb481eb71c121e040126d8 2024-11-10T16:13:47,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d30555385eba461c88cf799285393612 2024-11-10T16:13:47,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e10f3d54bbfb495c9307b409010fe20c 2024-11-10T16:13:47,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e67063d5d58c450bb7b5a270c4296425 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e67063d5d58c450bb7b5a270c4296425 2024-11-10T16:13:47,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/2dc0e028449541ee9d6e379968cf4772 2024-11-10T16:13:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33245f3b657b42c4a4de0e7916be0ff5 2024-11-10T16:13:47,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f09d20a7affc4526b4ed41a7beaf5ab3 2024-11-10T16:13:47,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33bd2edf0f074c09ac278cccf1644e17 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/33bd2edf0f074c09ac278cccf1644e17 2024-11-10T16:13:47,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/c7e7bb1423f34191a6e24075407de97a 2024-11-10T16:13:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/8662e3f2d8b94d288f293f0cc74ee898 2024-11-10T16:13:47,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/64a42213846249128e541c64461564db to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/64a42213846249128e541c64461564db 2024-11-10T16:13:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/d2cf08a4d3184764a861d7c66c69e3b0 2024-11-10T16:13:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/0364bf27c2844cedaefa1b443a997d20 2024-11-10T16:13:47,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/34fdb9bbbcf24275ad555af8a79770ce 2024-11-10T16:13:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/90b3aa461dd341ffb31926ff4027228b 2024-11-10T16:13:47,280 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/recovered.edits/506.seqid, newMaxSeqId=506, maxSeqId=1 2024-11-10T16:13:47,281 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. 2024-11-10T16:13:47,281 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for c8fb0b94c7d60b2a8bd105a3337e427b: Waiting for close lock at 1731255225874Running coprocessor pre-close hooks at 1731255225874Disabling compacts and flushes for region at 1731255225874Disabling writes for close at 1731255225874Obtaining lock to block concurrent updates at 1731255225874Preparing flush snapshotting stores in c8fb0b94c7d60b2a8bd105a3337e427b at 1731255225874Finished memstore snapshotting TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731255225875 (+1 ms)Flushing stores of TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b. at 1731255225875Flushing c8fb0b94c7d60b2a8bd105a3337e427b/A: creating writer at 1731255225876 (+1 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/A: appending metadata at 1731255225879 (+3 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/A: closing flushed file at 1731255225879Flushing c8fb0b94c7d60b2a8bd105a3337e427b/B: creating writer at 1731255226288 (+409 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/B: appending metadata at 1731255226291 (+3 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/B: closing flushed file at 1731255226291Flushing c8fb0b94c7d60b2a8bd105a3337e427b/C: creating writer at 1731255226701 (+410 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/C: appending metadata at 1731255226703 (+2 ms)Flushing c8fb0b94c7d60b2a8bd105a3337e427b/C: closing flushed file at 1731255226703Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f0df41f: reopening flushed file at 1731255227112 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d505c70: reopening flushed file at 1731255227117 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@483b26d6: reopening flushed file at 1731255227122 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c8fb0b94c7d60b2a8bd105a3337e427b in 1254ms, sequenceid=503, compaction requested=true at 1731255227128 (+6 ms)Writing region close event to WAL at 1731255227276 (+148 ms)Running coprocessor post-close hooks at 1731255227281 (+5 ms)Closed at 1731255227281 2024-11-10T16:13:47,284 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:47,284 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=c8fb0b94c7d60b2a8bd105a3337e427b, regionState=CLOSED 2024-11-10T16:13:47,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:13:47,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T16:13:47,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure c8fb0b94c7d60b2a8bd105a3337e427b, server=8e3de9160f9f,44105,1731255200149 in 1.5700 sec 2024-11-10T16:13:47,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T16:13:47,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8fb0b94c7d60b2a8bd105a3337e427b, UNASSIGN in 1.5760 sec 2024-11-10T16:13:47,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-10T16:13:47,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5830 sec 2024-11-10T16:13:47,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255227295"}]},"ts":"1731255227295"} 2024-11-10T16:13:47,296 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:13:47,296 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:13:47,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-10T16:13:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T16:13:47,853 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:47,857 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:13:47,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,862 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T16:13:47,864 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,871 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:47,875 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/recovered.edits] 2024-11-10T16:13:47,878 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7ca36a4224ca4038bd478533e8934b6d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/7ca36a4224ca4038bd478533e8934b6d 2024-11-10T16:13:47,880 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/94b481189892452b9654394cffe7b710 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/94b481189892452b9654394cffe7b710 2024-11-10T16:13:47,881 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c2f2be8864894514ae6b7d36bf955afa to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/c2f2be8864894514ae6b7d36bf955afa 2024-11-10T16:13:47,883 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e5877d7185d64b0685da74e860ba62d3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/A/e5877d7185d64b0685da74e860ba62d3 2024-11-10T16:13:47,885 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/20a73feefdec485aaee7a87115004ca4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/20a73feefdec485aaee7a87115004ca4 2024-11-10T16:13:47,887 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/329b3a26a29b4d998a16661238248ad0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/329b3a26a29b4d998a16661238248ad0 2024-11-10T16:13:47,888 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/60f53a5a54bc48218e144342cd76958c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/60f53a5a54bc48218e144342cd76958c 2024-11-10T16:13:47,889 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cbd984de27b34b9d9f81e38c7bbd86bb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/B/cbd984de27b34b9d9f81e38c7bbd86bb 2024-11-10T16:13:47,892 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/4658cd76c4a94fe9a024f92968789863 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/4658cd76c4a94fe9a024f92968789863 2024-11-10T16:13:47,893 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e389ffdcf76243b4970da76b161bf456 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e389ffdcf76243b4970da76b161bf456 2024-11-10T16:13:47,895 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e796cb48e277440d9bbcc2d2ffca9d01 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/e796cb48e277440d9bbcc2d2ffca9d01 2024-11-10T16:13:47,896 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4292fab5aba49caa82c80c59ac53a44 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/C/f4292fab5aba49caa82c80c59ac53a44 2024-11-10T16:13:47,899 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/recovered.edits/506.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b/recovered.edits/506.seqid 2024-11-10T16:13:47,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/c8fb0b94c7d60b2a8bd105a3337e427b 2024-11-10T16:13:47,900 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:13:47,907 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T16:13:47,920 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:13:47,952 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:13:47,953 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:13:47,954 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255227954"}]},"ts":"9223372036854775807"} 2024-11-10T16:13:47,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:13:47,958 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c8fb0b94c7d60b2a8bd105a3337e427b, NAME => 'TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:13:47,958 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:13:47,958 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255227958"}]},"ts":"9223372036854775807"} 2024-11-10T16:13:47,960 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:13:47,961 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 104 msec 2024-11-10T16:13:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T16:13:47,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:13:47,973 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:47,986 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=245 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286154732_22 at /127.0.0.1:49712 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286154732_22 at /127.0.0.1:59612 [Waiting for operation #289] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8e3de9160f9f:44105-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_75357217_22 at /127.0.0.1:50042 [Waiting for operation #153] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 339) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7207 (was 7728) 2024-11-10T16:13:47,994 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=245, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=7207 2024-11-10T16:13:47,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:13:47,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:13:47,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:47,998 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:13:47,998 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:47,999 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-10T16:13:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:47,999 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:13:48,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741951_1127 (size=1061) 2024-11-10T16:13:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:48,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T16:13:48,408 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:13:48,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741952_1128 (size=53) 2024-11-10T16:13:48,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 54d3f36a627b699150cc523f346a55d2, disabling compactions & flushes 2024-11-10T16:13:48,814 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. after waiting 0 ms 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:48,814 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:48,814 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54d3f36a627b699150cc523f346a55d2: Waiting for close lock at 1731255228814Disabling compacts and flushes for region at 1731255228814Disabling writes for close at 1731255228814Writing region close event to WAL at 1731255228814Closed at 1731255228814 2024-11-10T16:13:48,816 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:13:48,816 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255228816"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255228816"}]},"ts":"1731255228816"} 2024-11-10T16:13:48,818 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:13:48,818 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:13:48,819 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255228819"}]},"ts":"1731255228819"} 2024-11-10T16:13:48,820 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:13:48,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, ASSIGN}] 2024-11-10T16:13:48,821 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, ASSIGN 2024-11-10T16:13:48,822 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:13:48,973 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:48,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, ASSIGN because future has completed 2024-11-10T16:13:48,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:49,131 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:49,131 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:13:49,132 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,132 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:49,132 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,132 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,133 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:49,134 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:49,135 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName A 2024-11-10T16:13:49,135 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:49,136 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:49,136 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,137 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:49,137 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName B 2024-11-10T16:13:49,137 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:49,138 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:49,138 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,139 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:49,139 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName C 2024-11-10T16:13:49,139 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:49,139 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:49,139 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:49,139 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,140 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,140 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,141 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,141 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,142 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:13:49,142 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,144 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:13:49,145 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 54d3f36a627b699150cc523f346a55d2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62326654, jitterRate=-0.0712604820728302}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:13:49,145 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:49,146 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 54d3f36a627b699150cc523f346a55d2: Running coprocessor pre-open hook at 1731255229132Writing region info on filesystem at 1731255229132Initializing all the Stores at 1731255229133 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255229133Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255229133Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255229133Cleaning up temporary data from old regions at 1731255229141 (+8 ms)Running coprocessor post-open hooks at 1731255229145 (+4 ms)Region opened successfully at 1731255229146 (+1 ms) 2024-11-10T16:13:49,146 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., pid=36, masterSystemTime=1731255229128 2024-11-10T16:13:49,148 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:49,148 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:49,149 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:49,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:13:49,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-10T16:13:49,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 in 175 msec 2024-11-10T16:13:49,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T16:13:49,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, ASSIGN in 333 msec 2024-11-10T16:13:49,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:13:49,156 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255229156"}]},"ts":"1731255229156"} 2024-11-10T16:13:49,158 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:13:49,158 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:13:49,159 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1620 sec 2024-11-10T16:13:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-10T16:13:50,154 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:50,155 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:50,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:50,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:50,157 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251f0553, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:50,159 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f1c755, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:50,160 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:50,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:50,163 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:50,164 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:13:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:13:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ad5e6af 2024-11-10T16:13:50,165 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:13:50,166 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:13:50,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:13:50,171 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:13:50,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T16:13:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741953_1129 (size=1097) 2024-11-10T16:13:50,592 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T16:13:50,592 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T16:13:50,596 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:13:50,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE}] 2024-11-10T16:13:50,603 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE 2024-11-10T16:13:50,604 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:50,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE because future has completed 2024-11-10T16:13:50,606 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:13:50,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:50,759 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:50,759 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:13:50,759 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 54d3f36a627b699150cc523f346a55d2, disabling compactions & flushes 2024-11-10T16:13:50,759 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:50,759 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:50,759 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. after waiting 0 ms 2024-11-10T16:13:50,759 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:50,763 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T16:13:50,764 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:50,764 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 54d3f36a627b699150cc523f346a55d2: Waiting for close lock at 1731255230759Running coprocessor pre-close hooks at 1731255230759Disabling compacts and flushes for region at 1731255230759Disabling writes for close at 1731255230759Writing region close event to WAL at 1731255230760 (+1 ms)Running coprocessor post-close hooks at 1731255230764 (+4 ms)Closed at 1731255230764 2024-11-10T16:13:50,764 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 54d3f36a627b699150cc523f346a55d2 to self. 2024-11-10T16:13:50,766 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:50,766 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=CLOSED 2024-11-10T16:13:50,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:13:50,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-10T16:13:50,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 in 163 msec 2024-11-10T16:13:50,771 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=true 2024-11-10T16:13:50,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:50,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE because future has completed 2024-11-10T16:13:50,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:13:51,079 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,079 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:13:51,080 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,080 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:13:51,080 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,080 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,084 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,084 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:51,089 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName A 2024-11-10T16:13:51,091 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:51,091 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:51,092 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,092 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:51,092 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName B 2024-11-10T16:13:51,092 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:51,093 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:51,093 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,093 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:13:51,094 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54d3f36a627b699150cc523f346a55d2 columnFamilyName C 2024-11-10T16:13:51,094 DEBUG [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:51,094 INFO [StoreOpener-54d3f36a627b699150cc523f346a55d2-1 {}] regionserver.HStore(327): Store=54d3f36a627b699150cc523f346a55d2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:13:51,094 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,094 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,095 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,096 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,097 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,097 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,097 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:13:51,098 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,099 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 54d3f36a627b699150cc523f346a55d2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72645985, jitterRate=0.08250953257083893}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:13:51,099 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,099 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 54d3f36a627b699150cc523f346a55d2: Running coprocessor pre-open hook at 1731255231080Writing region info on filesystem at 1731255231080Initializing all the Stores at 1731255231081 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255231081Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255231081Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255231081Cleaning up temporary data from old regions at 1731255231097 (+16 ms)Running coprocessor post-open hooks at 1731255231099 (+2 ms)Region opened successfully at 1731255231099 2024-11-10T16:13:51,100 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., pid=41, masterSystemTime=1731255231076 2024-11-10T16:13:51,102 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,102 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,102 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:13:51,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-10T16:13:51,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 in 181 msec 2024-11-10T16:13:51,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T16:13:51,108 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, REOPEN/MOVE in 503 msec 2024-11-10T16:13:51,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-10T16:13:51,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-11-10T16:13:51,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-10T16:13:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-10T16:13:51,117 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:51,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,119 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4083300d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,121 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f90a6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,124 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,125 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,125 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35732c07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208b436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,127 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,127 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,127 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f524406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,129 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6386de62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@349085e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4021cc61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,132 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,132 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8ac781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d680f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdedd03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,136 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d21c225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,136 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,136 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,137 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6563e248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553e52c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a92e196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,140 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@719d17f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,140 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,141 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621343d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3806f988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,143 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79aee5b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,143 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,144 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef014ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@494eb83f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:13:51,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:13:51,145 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6287ad98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:13:51,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:13:51,147 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:13:51,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62990a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:13:51,148 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,148 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,148 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,148 DEBUG [Thread-658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,148 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,148 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,149 DEBUG [Thread-653 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,150 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,150 DEBUG [Thread-653 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,150 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,151 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:51,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T16:13:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,153 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:13:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,154 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:51,154 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:13:51,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:13:51,155 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:51,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:51,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:13:51,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:13:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:51,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:51,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111047b33bf9cc794a828569923739633e09_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_1/A:col10/1731255231163/Put/seqid=0 2024-11-10T16:13:51,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255291216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255231196, max timestamp=1731255231216Num uniques 859; 2024-11-10T16:13:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:13:51,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255231196, max timestamp=1731255231219Num uniques 859; 2024-11-10T16:13:51,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:13:51,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731255231196, max timestamp=1731255231216Num uniques 859; 2024-11-10T16:13:51,231 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T16:13:51,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741954_1130 (size=19474) 2024-11-10T16:13:51,241 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:51,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111047b33bf9cc794a828569923739633e09_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047b33bf9cc794a828569923739633e09_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,249 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b4393ec29d8048b3b7766164d64ec285, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:51,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b4393ec29d8048b3b7766164d64ec285 is 175, key is test_row_1/A:col10/1731255231163/Put/seqid=0 2024-11-10T16:13:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:51,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255291274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741955_1131 (size=56729) 2024-11-10T16:13:51,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255231266, max timestamp=1731255231276Num uniques 1009; 2024-11-10T16:13:51,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-11-10T16:13:51,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255231266, max timestamp=1731255231276Num uniques 1018; 2024-11-10T16:13:51,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T16:13:51,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255231266, max timestamp=1731255231276Num uniques 1018; 2024-11-10T16:13:51,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T16:13:51,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:51,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255291344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255291344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255291346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255291348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255291384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255291455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255291455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255291455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255291463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255291593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:51,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:51,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255291663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255291663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255291663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255291674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:51,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:51,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b4393ec29d8048b3b7766164d64ec285 2024-11-10T16:13:51,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/e55684ea241148ecb60f6a30c06ad1ac is 50, key is test_row_1/B:col10/1731255231163/Put/seqid=0 2024-11-10T16:13:51,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741956_1132 (size=9657) 2024-11-10T16:13:51,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/e55684ea241148ecb60f6a30c06ad1ac 2024-11-10T16:13:51,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/284de5619cad4434a76351e2af149324 is 50, key is test_row_1/C:col10/1731255231163/Put/seqid=0 2024-11-10T16:13:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:51,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741957_1133 (size=9657) 2024-11-10T16:13:51,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/284de5619cad4434a76351e2af149324 2024-11-10T16:13:51,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b4393ec29d8048b3b7766164d64ec285 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285 2024-11-10T16:13:51,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285, entries=300, sequenceid=14, filesize=55.4 K 2024-11-10T16:13:51,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/e55684ea241148ecb60f6a30c06ad1ac as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac 2024-11-10T16:13:51,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T16:13:51,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/284de5619cad4434a76351e2af149324 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324 2024-11-10T16:13:51,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T16:13:51,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 54d3f36a627b699150cc523f346a55d2 in 654ms, sequenceid=14, compaction requested=false 2024-11-10T16:13:51,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731255231323, max timestamp=1731255231907Num uniques 1027; 2024-11-10T16:13:51,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T16:13:51,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731255231323, max timestamp=1731255231907Num uniques 1027; 2024-11-10T16:13:51,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T16:13:51,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731255231323, max timestamp=1731255231907Num uniques 1027; 2024-11-10T16:13:51,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T16:13:51,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=26.84 KB heapSize=64.55 KB 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:51,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:51,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f24883d4a60f40868397733cc0cc5c5f_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255231345/Put/seqid=0 2024-11-10T16:13:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:51,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:51,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741958_1134 (size=14594) 2024-11-10T16:13:51,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:51,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f24883d4a60f40868397733cc0cc5c5f_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f24883d4a60f40868397733cc0cc5c5f_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:51,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/582d8abc0edb49da937f6f6a69165f47, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:51,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/582d8abc0edb49da937f6f6a69165f47 is 175, key is test_row_0/A:col10/1731255231345/Put/seqid=0 2024-11-10T16:13:51,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741959_1135 (size=39549) 2024-11-10T16:13:51,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/582d8abc0edb49da937f6f6a69165f47 2024-11-10T16:13:51,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a2b80f57e1dc4aefbd2ee4d02199b723 is 50, key is test_row_0/B:col10/1731255231345/Put/seqid=0 2024-11-10T16:13:52,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255231991, max timestamp=1731255232001Num uniques 859; 2024-11-10T16:13:52,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255231991, max timestamp=1731255232001Num uniques 859; 2024-11-10T16:13:52,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:13:52,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:13:52,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255231991, max timestamp=1731255232001Num uniques 859; 2024-11-10T16:13:52,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:13:52,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741960_1136 (size=12001) 2024-11-10T16:13:52,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a2b80f57e1dc4aefbd2ee4d02199b723 2024-11-10T16:13:52,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9f714f48b2f34ea38a1a4fa6c214d306 is 50, key is test_row_0/C:col10/1731255231345/Put/seqid=0 2024-11-10T16:13:52,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741961_1137 (size=12001) 2024-11-10T16:13:52,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9f714f48b2f34ea38a1a4fa6c214d306 2024-11-10T16:13:52,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/582d8abc0edb49da937f6f6a69165f47 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47 2024-11-10T16:13:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731255232031, max timestamp=1731255232048Num uniques 1068; 2024-11-10T16:13:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731255232031, max timestamp=1731255232048Num uniques 1068; 2024-11-10T16:13:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T16:13:52,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T16:13:52,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731255232031, max timestamp=1731255232048Num uniques 1068; 2024-11-10T16:13:52,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-10T16:13:52,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47, entries=200, sequenceid=69, filesize=38.6 K 2024-11-10T16:13:52,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a2b80f57e1dc4aefbd2ee4d02199b723 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723 2024-11-10T16:13:52,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T16:13:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9f714f48b2f34ea38a1a4fa6c214d306 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306 2024-11-10T16:13:52,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306, entries=150, sequenceid=69, filesize=11.7 K 2024-11-10T16:13:52,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~64.50 KB/66048, currentSize=154.31 KB/158010 for 54d3f36a627b699150cc523f346a55d2 in 179ms, sequenceid=69, compaction requested=false 2024-11-10T16:13:52,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:52,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:52,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731255232078, max timestamp=1731255232092Num uniques 1177; 2024-11-10T16:13:52,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-10T16:13:52,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731255232078, max timestamp=1731255232092Num uniques 1177; 2024-11-10T16:13:52,098 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-10T16:13:52,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=96.16 KB heapSize=263.11 KB 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:52,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T16:13:52,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110079f7a86d9d94c1daa790b1ba39c9da4_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741963_1139 (size=24358) 2024-11-10T16:13:52,151 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:52,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110079f7a86d9d94c1daa790b1ba39c9da4_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110079f7a86d9d94c1daa790b1ba39c9da4_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:52,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c1daeef7ed6d425dbc61cf9341a1b4ab, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c1daeef7ed6d425dbc61cf9341a1b4ab is 175, key is test_row_0/A:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741962_1138 (size=73995) 2024-11-10T16:13:52,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c1daeef7ed6d425dbc61cf9341a1b4ab 2024-11-10T16:13:52,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/473482320aca4cb8bbcb800dcbd87eb3 is 50, key is test_row_0/B:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255292158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255292163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255292161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255292183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255292184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741964_1140 (size=12001) 2024-11-10T16:13:52,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/473482320aca4cb8bbcb800dcbd87eb3 2024-11-10T16:13:52,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/64a89f0c74f143e7b7320e4aaa5c95a6 is 50, key is test_row_0/C:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741965_1141 (size=12001) 2024-11-10T16:13:52,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255292294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255292294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255292294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255292295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255292295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255292504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255292504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255292504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,505 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,506 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255292513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255292513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:52,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/64a89f0c74f143e7b7320e4aaa5c95a6 2024-11-10T16:13:52,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c1daeef7ed6d425dbc61cf9341a1b4ab as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab 2024-11-10T16:13:52,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab, entries=400, sequenceid=127, filesize=72.3 K 2024-11-10T16:13:52,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/473482320aca4cb8bbcb800dcbd87eb3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3 2024-11-10T16:13:52,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3, entries=150, sequenceid=127, filesize=11.7 K 2024-11-10T16:13:52,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/64a89f0c74f143e7b7320e4aaa5c95a6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6 2024-11-10T16:13:52,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6, entries=150, sequenceid=127, filesize=11.7 K 2024-11-10T16:13:52,667 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~96.16 KB/98470, heapSize ~263.06 KB/269376, currentSize=93.93 KB/96180 for 54d3f36a627b699150cc523f346a55d2 in 558ms, sequenceid=127, compaction requested=true 2024-11-10T16:13:52,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:52,668 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:52,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:52,669 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 170273 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:52,669 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:13:52,669 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,669 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=166.3 K 2024-11-10T16:13:52,670 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:52,670 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,670 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab] 2024-11-10T16:13:52,671 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4393ec29d8048b3b7766164d64ec285, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255231158 2024-11-10T16:13:52,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:52,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:13:52,671 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,671 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=32.9 K 2024-11-10T16:13:52,672 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e55684ea241148ecb60f6a30c06ad1ac, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255231163 2024-11-10T16:13:52,672 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 582d8abc0edb49da937f6f6a69165f47, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731255231323 2024-11-10T16:13:52,672 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a2b80f57e1dc4aefbd2ee4d02199b723, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731255231343 2024-11-10T16:13:52,672 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1daeef7ed6d425dbc61cf9341a1b4ab, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232078 2024-11-10T16:13:52,673 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 473482320aca4cb8bbcb800dcbd87eb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232090 2024-11-10T16:13:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:52,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:52,693 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,697 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:52,698 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/ffa22081990a4ef381969f4dc3242fe4 is 50, key is test_row_0/B:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T16:13:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,699 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:13:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:52,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,709 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108b755bba7b7b4ae7a6b944a389a3a18c_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,716 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108b755bba7b7b4ae7a6b944a389a3a18c_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,717 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108b755bba7b7b4ae7a6b944a389a3a18c_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741966_1142 (size=12104) 2024-11-10T16:13:52,749 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/ffa22081990a4ef381969f4dc3242fe4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/ffa22081990a4ef381969f4dc3242fe4 2024-11-10T16:13:52,755 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into ffa22081990a4ef381969f4dc3242fe4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:52,755 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:52,755 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255232670; duration=0sec 2024-11-10T16:13:52,755 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:52,755 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:13:52,756 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:52,757 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:52,758 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:13:52,758 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,758 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=32.9 K 2024-11-10T16:13:52,759 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 284de5619cad4434a76351e2af149324, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255231163 2024-11-10T16:13:52,760 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f714f48b2f34ea38a1a4fa6c214d306, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731255231343 2024-11-10T16:13:52,761 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a89f0c74f143e7b7320e4aaa5c95a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232090 2024-11-10T16:13:52,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741967_1143 (size=4469) 2024-11-10T16:13:52,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f3de4823e7124637a4517291977a6a0a_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255232160/Put/seqid=0 2024-11-10T16:13:52,789 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#126 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:52,790 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/df5a495975164396bfe2940c38d3820a is 50, key is test_row_0/C:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:52,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741968_1144 (size=12304) 2024-11-10T16:13:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:52,805 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f3de4823e7124637a4517291977a6a0a_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3de4823e7124637a4517291977a6a0a_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:52,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f13406b383b14b8986d55af297235208, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:52,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f13406b383b14b8986d55af297235208 is 175, key is test_row_0/A:col10/1731255232160/Put/seqid=0 2024-11-10T16:13:52,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741969_1145 (size=12104) 2024-11-10T16:13:52,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:52,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:52,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/df5a495975164396bfe2940c38d3820a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/df5a495975164396bfe2940c38d3820a 2024-11-10T16:13:52,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:13:52,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into df5a495975164396bfe2940c38d3820a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:52,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:52,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=13, startTime=1731255232673; duration=0sec 2024-11-10T16:13:52,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:52,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:13:52,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741970_1146 (size=31105) 2024-11-10T16:13:52,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f13406b383b14b8986d55af297235208 2024-11-10T16:13:52,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255292845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255292847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255292848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255292850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255292851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/496b46cf84f040d5968f55a30f2ad8d1 is 50, key is test_row_0/B:col10/1731255232160/Put/seqid=0 2024-11-10T16:13:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741971_1147 (size=12151) 2024-11-10T16:13:52,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/496b46cf84f040d5968f55a30f2ad8d1 2024-11-10T16:13:52,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/8745799a43da49fba0613c3c051b7ccf is 50, key is test_row_0/C:col10/1731255232160/Put/seqid=0 2024-11-10T16:13:52,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741972_1148 (size=12151) 2024-11-10T16:13:52,904 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/8745799a43da49fba0613c3c051b7ccf 2024-11-10T16:13:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f13406b383b14b8986d55af297235208 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208 2024-11-10T16:13:52,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208, entries=150, sequenceid=144, filesize=30.4 K 2024-11-10T16:13:52,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/496b46cf84f040d5968f55a30f2ad8d1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1 2024-11-10T16:13:52,936 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T16:13:52,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/8745799a43da49fba0613c3c051b7ccf as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf 2024-11-10T16:13:52,942 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T16:13:52,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 54d3f36a627b699150cc523f346a55d2 in 244ms, sequenceid=144, compaction requested=false 2024-11-10T16:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T16:13:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T16:13:52,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T16:13:52,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2024-11-10T16:13:52,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7970 sec 2024-11-10T16:13:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:52,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:52,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:52,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255292981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255292985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255292986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255292988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255292988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:52,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110167d147ed6f740f58026e3144d147698_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:53,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741973_1149 (size=17284) 2024-11-10T16:13:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255293093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255293093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255293103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255293104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255293104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,170 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#123 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:53,171 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/652616392aae4b7483491c153a4ad90c is 175, key is test_row_0/A:col10/1731255232090/Put/seqid=0 2024-11-10T16:13:53,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741974_1150 (size=31058) 2024-11-10T16:13:53,185 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/652616392aae4b7483491c153a4ad90c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c 2024-11-10T16:13:53,196 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into 652616392aae4b7483491c153a4ad90c(size=30.3 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:53,196 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:53,196 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255232667; duration=0sec 2024-11-10T16:13:53,196 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:53,196 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:13:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T16:13:53,303 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:53,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T16:13:53,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T16:13:53,307 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:53,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255293303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255293303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,311 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:53,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255293314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255293314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255293314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,411 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T16:13:53,417 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110167d147ed6f740f58026e3144d147698_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167d147ed6f740f58026e3144d147698_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:53,418 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fdf289632da24ee48ac4b0f3a3a97fba, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:53,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fdf289632da24ee48ac4b0f3a3a97fba is 175, key is test_row_0/A:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:53,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741975_1151 (size=48389) 2024-11-10T16:13:53,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fdf289632da24ee48ac4b0f3a3a97fba 2024-11-10T16:13:53,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/f8ef7ef05d3946ebb7f905f97160e9ca is 50, key is test_row_0/B:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:53,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T16:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741976_1152 (size=12151) 2024-11-10T16:13:53,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255293613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255293613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T16:13:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T16:13:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255293623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255293623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255293623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:53,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:53,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T16:13:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/f8ef7ef05d3946ebb7f905f97160e9ca 2024-11-10T16:13:53,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/d72865e876f841d1bea94737e2b74ea1 is 50, key is test_row_0/C:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741977_1153 (size=12151) 2024-11-10T16:13:53,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/d72865e876f841d1bea94737e2b74ea1 2024-11-10T16:13:53,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fdf289632da24ee48ac4b0f3a3a97fba as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba 2024-11-10T16:13:53,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba, entries=250, sequenceid=163, filesize=47.3 K 2024-11-10T16:13:53,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/f8ef7ef05d3946ebb7f905f97160e9ca as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca 2024-11-10T16:13:53,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T16:13:53,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/d72865e876f841d1bea94737e2b74ea1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1 2024-11-10T16:13:53,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T16:13:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:53,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1, entries=150, sequenceid=163, filesize=11.9 K 2024-11-10T16:13:53,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 54d3f36a627b699150cc523f346a55d2 in 962ms, sequenceid=163, compaction requested=true 2024-11-10T16:13:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:53,928 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:53,928 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:53,930 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:53,930 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110552 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:53,930 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:13:53,930 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:13:53,930 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,930 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,930 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=108.0 K 2024-11-10T16:13:53,930 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/ffa22081990a4ef381969f4dc3242fe4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=35.6 K 2024-11-10T16:13:53,930 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:53,930 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba] 2024-11-10T16:13:53,931 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652616392aae4b7483491c153a4ad90c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232090 2024-11-10T16:13:53,931 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting ffa22081990a4ef381969f4dc3242fe4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232090 2024-11-10T16:13:53,932 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f13406b383b14b8986d55af297235208, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731255232158 2024-11-10T16:13:53,932 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 496b46cf84f040d5968f55a30f2ad8d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731255232158 2024-11-10T16:13:53,932 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ef7ef05d3946ebb7f905f97160e9ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731255232846 2024-11-10T16:13:53,932 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf289632da24ee48ac4b0f3a3a97fba, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731255232844 2024-11-10T16:13:53,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T16:13:53,940 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:53,942 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:53,942 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/75247361850f4c1395ede38c32432131 is 50, key is test_row_0/B:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:53,942 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b2bd1149179248078a31b9785ef1c6be_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:53,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741978_1154 (size=12357) 2024-11-10T16:13:53,952 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b2bd1149179248078a31b9785ef1c6be_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:53,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2bd1149179248078a31b9785ef1c6be_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:53,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741979_1155 (size=4469) 2024-11-10T16:13:54,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T16:13:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,078 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:13:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e0fdaa60da364158bd1bb06cd7215cfb_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255232975/Put/seqid=0 2024-11-10T16:13:54,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741980_1156 (size=12304) 2024-11-10T16:13:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:54,120 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e0fdaa60da364158bd1bb06cd7215cfb_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e0fdaa60da364158bd1bb06cd7215cfb_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/a1288cc998c84f1d851e1c4da4e5bd83, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/a1288cc998c84f1d851e1c4da4e5bd83 is 175, key is test_row_0/A:col10/1731255232975/Put/seqid=0 2024-11-10T16:13:54,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741981_1157 (size=31105) 2024-11-10T16:13:54,132 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/a1288cc998c84f1d851e1c4da4e5bd83 2024-11-10T16:13:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2af72f7443e84d0996a95b8aa9d522d8 is 50, key is test_row_0/B:col10/1731255232975/Put/seqid=0 2024-11-10T16:13:54,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255294156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741982_1158 (size=12151) 2024-11-10T16:13:54,167 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2af72f7443e84d0996a95b8aa9d522d8 2024-11-10T16:13:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255294162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255294162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255294163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255294164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/0798f75e63b744e0b6257b6a95fce67a is 50, key is test_row_0/C:col10/1731255232975/Put/seqid=0 2024-11-10T16:13:54,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741983_1159 (size=12151) 2024-11-10T16:13:54,191 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/0798f75e63b744e0b6257b6a95fce67a 2024-11-10T16:13:54,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/a1288cc998c84f1d851e1c4da4e5bd83 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83 2024-11-10T16:13:54,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83, entries=150, sequenceid=179, filesize=30.4 K 2024-11-10T16:13:54,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2af72f7443e84d0996a95b8aa9d522d8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8 2024-11-10T16:13:54,216 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8, entries=150, sequenceid=179, filesize=11.9 K 2024-11-10T16:13:54,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/0798f75e63b744e0b6257b6a95fce67a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a 2024-11-10T16:13:54,225 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a, entries=150, sequenceid=179, filesize=11.9 K 2024-11-10T16:13:54,226 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 54d3f36a627b699150cc523f346a55d2 in 148ms, sequenceid=179, compaction requested=true 2024-11-10T16:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T16:13:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T16:13:54,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T16:13:54,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 918 msec 2024-11-10T16:13:54,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 928 msec 2024-11-10T16:13:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:54,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cdf282dcba28490ea96afe71cfb2d22e_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255294293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741984_1160 (size=14794) 2024-11-10T16:13:54,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255294295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255294296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255294297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255294298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,352 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/75247361850f4c1395ede38c32432131 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/75247361850f4c1395ede38c32432131 2024-11-10T16:13:54,365 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into 75247361850f4c1395ede38c32432131(size=12.1 K), total size for store is 23.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:54,365 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:54,365 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255233928; duration=0sec 2024-11-10T16:13:54,365 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:54,365 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:13:54,365 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:54,367 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48557 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:54,367 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:13:54,367 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,367 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/df5a495975164396bfe2940c38d3820a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=47.4 K 2024-11-10T16:13:54,367 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#133 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:54,368 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c407973812c245b98810d5c02775025b is 175, key is test_row_0/A:col10/1731255232846/Put/seqid=0 2024-11-10T16:13:54,369 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting df5a495975164396bfe2940c38d3820a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1731255232090 2024-11-10T16:13:54,370 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8745799a43da49fba0613c3c051b7ccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731255232158 2024-11-10T16:13:54,372 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d72865e876f841d1bea94737e2b74ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731255232846 2024-11-10T16:13:54,372 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0798f75e63b744e0b6257b6a95fce67a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731255232975 2024-11-10T16:13:54,392 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:54,392 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/25165f9beb02406db4901370a419fadd is 50, key is test_row_0/C:col10/1731255232975/Put/seqid=0 2024-11-10T16:13:54,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741985_1161 (size=31311) 2024-11-10T16:13:54,402 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/c407973812c245b98810d5c02775025b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b 2024-11-10T16:13:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255294404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255294404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,410 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,411 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into c407973812c245b98810d5c02775025b(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:54,411 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:54,411 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255233928; duration=0sec 2024-11-10T16:13:54,411 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:54,411 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:13:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255294413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255294413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255294413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741986_1162 (size=12391) 2024-11-10T16:13:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T16:13:54,443 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:54,443 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T16:13:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T16:13:54,445 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:54,446 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:54,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T16:13:54,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T16:13:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255294613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255294623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255294624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255294624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255294624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:54,700 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:54,705 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cdf282dcba28490ea96afe71cfb2d22e_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cdf282dcba28490ea96afe71cfb2d22e_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,706 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b766f413ebc45348c9edd176cd3ab64, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b766f413ebc45348c9edd176cd3ab64 is 175, key is test_row_0/A:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741987_1163 (size=39749) 2024-11-10T16:13:54,711 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b766f413ebc45348c9edd176cd3ab64 2024-11-10T16:13:54,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/92f38bfd9b0c465ba1804af09369ec7e is 50, key is test_row_0/B:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741988_1164 (size=12151) 2024-11-10T16:13:54,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/92f38bfd9b0c465ba1804af09369ec7e 2024-11-10T16:13:54,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T16:13:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:54,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/94be0d5e7ff343a0aa90c931611bba92 is 50, key is test_row_0/C:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T16:13:54,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741989_1165 (size=12151) 2024-11-10T16:13:54,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/94be0d5e7ff343a0aa90c931611bba92 2024-11-10T16:13:54,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b766f413ebc45348c9edd176cd3ab64 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64 2024-11-10T16:13:54,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64, entries=200, sequenceid=198, filesize=38.8 K 2024-11-10T16:13:54,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/92f38bfd9b0c465ba1804af09369ec7e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e 2024-11-10T16:13:54,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e, entries=150, sequenceid=198, filesize=11.9 K 2024-11-10T16:13:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/94be0d5e7ff343a0aa90c931611bba92 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92 2024-11-10T16:13:54,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92, entries=150, sequenceid=198, filesize=11.9 K 2024-11-10T16:13:54,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 54d3f36a627b699150cc523f346a55d2 in 526ms, sequenceid=198, compaction requested=true 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:54,803 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:54,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:13:54,804 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102165 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:54,804 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:13:54,804 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,804 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=99.8 K 2024-11-10T16:13:54,804 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,804 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64] 2024-11-10T16:13:54,806 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c407973812c245b98810d5c02775025b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731255232846 2024-11-10T16:13:54,806 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1288cc998c84f1d851e1c4da4e5bd83, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731255232975 2024-11-10T16:13:54,806 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b766f413ebc45348c9edd176cd3ab64, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731255234159 2024-11-10T16:13:54,817 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,819 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109c8e040d5bef40ae9bd177e9ab497196_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,822 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109c8e040d5bef40ae9bd177e9ab497196_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,822 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109c8e040d5bef40ae9bd177e9ab497196_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,829 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/25165f9beb02406db4901370a419fadd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/25165f9beb02406db4901370a419fadd 2024-11-10T16:13:54,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741990_1166 (size=4469) 2024-11-10T16:13:54,836 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 25165f9beb02406db4901370a419fadd(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:54,836 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:54,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=12, startTime=1731255233930; duration=0sec 2024-11-10T16:13:54,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:13:54,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:13:54,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:13:54,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:54,838 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#141 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:54,839 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/693d3294764b4f088f6be8d3419a6b16 is 175, key is test_row_0/A:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,839 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:54,839 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:13:54,840 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,840 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/75247361850f4c1395ede38c32432131, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=35.8 K 2024-11-10T16:13:54,840 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 75247361850f4c1395ede38c32432131, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1731255232846 2024-11-10T16:13:54,841 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af72f7443e84d0996a95b8aa9d522d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731255232975 2024-11-10T16:13:54,841 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 92f38bfd9b0c465ba1804af09369ec7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731255234161 2024-11-10T16:13:54,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741991_1167 (size=31413) 2024-11-10T16:13:54,855 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:54,856 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d01ab44dc0ef4873b1e1f57e66fc0d49 is 50, key is test_row_0/B:col10/1731255234162/Put/seqid=0 2024-11-10T16:13:54,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741992_1168 (size=12459) 2024-11-10T16:13:54,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T16:13:54,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:54,907 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:54,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:54,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110347d65e57bca45adbe95cdafa838c0ea_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255234295/Put/seqid=0 2024-11-10T16:13:54,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741993_1169 (size=12304) 2024-11-10T16:13:54,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:54,937 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110347d65e57bca45adbe95cdafa838c0ea_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110347d65e57bca45adbe95cdafa838c0ea_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:54,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:54,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b59080acf24470392afdc11b8fab4c8, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b59080acf24470392afdc11b8fab4c8 is 175, key is test_row_0/A:col10/1731255234295/Put/seqid=0 2024-11-10T16:13:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741994_1170 (size=31105) 2024-11-10T16:13:54,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255294988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255294989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255294991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255294992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:54,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255294994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:54,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T16:13:55,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255295103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255295103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255295103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255295103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255295103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,260 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/693d3294764b4f088f6be8d3419a6b16 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16 2024-11-10T16:13:55,267 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into 693d3294764b4f088f6be8d3419a6b16(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:55,267 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:55,267 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255234803; duration=0sec 2024-11-10T16:13:55,267 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:55,267 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:13:55,268 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-10T16:13:55,269 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-10T16:13:55,269 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-10T16:13:55,269 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. because compaction request was cancelled 2024-11-10T16:13:55,269 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:13:55,288 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d01ab44dc0ef4873b1e1f57e66fc0d49 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d01ab44dc0ef4873b1e1f57e66fc0d49 2024-11-10T16:13:55,298 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into d01ab44dc0ef4873b1e1f57e66fc0d49(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:55,298 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:55,298 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255234803; duration=0sec 2024-11-10T16:13:55,298 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:55,298 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:13:55,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255295313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255295313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255295313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255295313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255295314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,357 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b59080acf24470392afdc11b8fab4c8 2024-11-10T16:13:55,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/66cbb7dfde82453f909733d1cbe1f342 is 50, key is test_row_0/B:col10/1731255234295/Put/seqid=0 2024-11-10T16:13:55,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741995_1171 (size=12151) 2024-11-10T16:13:55,374 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/66cbb7dfde82453f909733d1cbe1f342 2024-11-10T16:13:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/b7ad207e4080478a894267c528eeb1bf is 50, key is test_row_0/C:col10/1731255234295/Put/seqid=0 2024-11-10T16:13:55,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741996_1172 (size=12151) 2024-11-10T16:13:55,399 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/b7ad207e4080478a894267c528eeb1bf 2024-11-10T16:13:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0b59080acf24470392afdc11b8fab4c8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8 2024-11-10T16:13:55,416 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8, entries=150, sequenceid=214, filesize=30.4 K 2024-11-10T16:13:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/66cbb7dfde82453f909733d1cbe1f342 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342 2024-11-10T16:13:55,429 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T16:13:55,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/b7ad207e4080478a894267c528eeb1bf as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf 2024-11-10T16:13:55,438 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T16:13:55,439 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 54d3f36a627b699150cc523f346a55d2 in 532ms, sequenceid=214, compaction requested=true 2024-11-10T16:13:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T16:13:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T16:13:55,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T16:13:55,443 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-10T16:13:55,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0000 sec 2024-11-10T16:13:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T16:13:55,583 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:55,584 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T16:13:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:55,587 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:55,588 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:55,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:55,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T16:13:55,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:55,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:55,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:55,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:55,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:55,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:55,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102cd1b473e04e4b918b6a6cc5a2f112ac_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:55,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255295636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255295637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255295640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255295643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255295645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741997_1173 (size=14794) 2024-11-10T16:13:55,655 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:55,660 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102cd1b473e04e4b918b6a6cc5a2f112ac_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102cd1b473e04e4b918b6a6cc5a2f112ac_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:55,661 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/80034c09d5904413bee800f5d6b87ceb, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:55,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/80034c09d5904413bee800f5d6b87ceb is 175, key is test_row_0/A:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:55,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741998_1174 (size=39749) 2024-11-10T16:13:55,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/80034c09d5904413bee800f5d6b87ceb 2024-11-10T16:13:55,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c2aa365acb5741e39f6c3b02e055dd6d is 50, key is test_row_0/B:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:55,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741999_1175 (size=12151) 2024-11-10T16:13:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:55,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:55,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255295754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255295754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255295754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255295763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255295763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,766 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:55,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:55,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255295963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255295963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255295963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255295973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255295974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c2aa365acb5741e39f6c3b02e055dd6d 2024-11-10T16:13:56,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/cf0077d47ab84530b6603256e037ba8b is 50, key is test_row_0/C:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:56,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742000_1176 (size=12151) 2024-11-10T16:13:56,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255296273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255296273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255296273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255296284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255296284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:56,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/cf0077d47ab84530b6603256e037ba8b 2024-11-10T16:13:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/80034c09d5904413bee800f5d6b87ceb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb 2024-11-10T16:13:56,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:56,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:13:56,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb, entries=200, sequenceid=236, filesize=38.8 K 2024-11-10T16:13:56,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c2aa365acb5741e39f6c3b02e055dd6d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d 2024-11-10T16:13:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d, entries=150, sequenceid=236, filesize=11.9 K 2024-11-10T16:13:56,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/cf0077d47ab84530b6603256e037ba8b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b 2024-11-10T16:13:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b, entries=150, sequenceid=236, filesize=11.9 K 2024-11-10T16:13:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 54d3f36a627b699150cc523f346a55d2 in 901ms, sequenceid=236, compaction requested=true 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:56,529 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:56,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,529 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,530 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,531 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:56,531 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:13:56,531 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:13:56,531 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,531 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,531 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d01ab44dc0ef4873b1e1f57e66fc0d49, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=35.9 K 2024-11-10T16:13:56,531 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=99.9 K 2024-11-10T16:13:56,531 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,531 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb] 2024-11-10T16:13:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,532 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d01ab44dc0ef4873b1e1f57e66fc0d49, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731255234161 2024-11-10T16:13:56,532 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693d3294764b4f088f6be8d3419a6b16, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731255234161 2024-11-10T16:13:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,532 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 66cbb7dfde82453f909733d1cbe1f342, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255234291 2024-11-10T16:13:56,532 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b59080acf24470392afdc11b8fab4c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255234291 2024-11-10T16:13:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,533 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c2aa365acb5741e39f6c3b02e055dd6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234990 2024-11-10T16:13:56,533 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80034c09d5904413bee800f5d6b87ceb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234985 2024-11-10T16:13:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,541 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,543 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:56,543 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110dbfdf0dea7af4f92898953f65f533ade_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:56,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,543 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d24aee01fb874f4b85bb68b065740879 is 50, key is test_row_0/B:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,545 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110dbfdf0dea7af4f92898953f65f533ade_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,545 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dbfdf0dea7af4f92898953f65f533ade_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742002_1178 (size=4469) 2024-11-10T16:13:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742001_1177 (size=12561) 2024-11-10T16:13:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,553 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#150 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,554 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/4765b5eed5a84b8e977a10d0ea3f0cf8 is 175, key is test_row_0/A:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742003_1179 (size=31515) 2024-11-10T16:13:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,572 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/4765b5eed5a84b8e977a10d0ea3f0cf8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8 2024-11-10T16:13:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,578 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into 4765b5eed5a84b8e977a10d0ea3f0cf8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:56,578 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:56,578 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255236529; duration=0sec 2024-11-10T16:13:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,578 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:56,579 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:13:56,579 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:13:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,580 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:13:56,580 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:13:56,580 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,580 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/25165f9beb02406db4901370a419fadd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=47.7 K 2024-11-10T16:13:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,581 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25165f9beb02406db4901370a419fadd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731255232975 2024-11-10T16:13:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,581 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94be0d5e7ff343a0aa90c931611bba92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731255234161 2024-11-10T16:13:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,582 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7ad207e4080478a894267c528eeb1bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255234291 2024-11-10T16:13:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,582 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0077d47ab84530b6603256e037ba8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234990 2024-11-10T16:13:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,593 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,594 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1c7aeba4a5eb4fa58312aea1fcd50a40 is 50, key is test_row_0/C:col10/1731255234992/Put/seqid=0 2024-11-10T16:13:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742004_1180 (size=12527) 2024-11-10T16:13:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,628 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1c7aeba4a5eb4fa58312aea1fcd50a40 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1c7aeba4a5eb4fa58312aea1fcd50a40 2024-11-10T16:13:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,634 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 1c7aeba4a5eb4fa58312aea1fcd50a40(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:56,634 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:56,634 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=12, startTime=1731255236529; duration=0sec 2024-11-10T16:13:56,634 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:56,634 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:13:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T16:13:56,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,664 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:56,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e42dffda93ce4ad7aa120a830edc6d07_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255235641/Put/seqid=0 2024-11-10T16:13:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742005_1181 (size=9814) 2024-11-10T16:13:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:13:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255296818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255296818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255296819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255296820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255296821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255296934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:56,959 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d24aee01fb874f4b85bb68b065740879 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d24aee01fb874f4b85bb68b065740879 2024-11-10T16:13:56,971 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into d24aee01fb874f4b85bb68b065740879(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:56,971 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:56,971 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255236529; duration=0sec 2024-11-10T16:13:56,971 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:56,971 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:13:57,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:57,086 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e42dffda93ce4ad7aa120a830edc6d07_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e42dffda93ce4ad7aa120a830edc6d07_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:57,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e2fada4f92684d4aa9dc510e0bd7b939, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:57,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e2fada4f92684d4aa9dc510e0bd7b939 is 175, key is test_row_0/A:col10/1731255235641/Put/seqid=0 2024-11-10T16:13:57,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742006_1182 (size=22461) 2024-11-10T16:13:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255297143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255297143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255297143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255297143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255297143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255297453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255297454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e2fada4f92684d4aa9dc510e0bd7b939 2024-11-10T16:13:57,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/5a57d099dbec425a91200e38bcbf8614 is 50, key is test_row_0/B:col10/1731255235641/Put/seqid=0 2024-11-10T16:13:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742007_1183 (size=9757) 2024-11-10T16:13:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:57,906 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/5a57d099dbec425a91200e38bcbf8614 2024-11-10T16:13:57,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5b8dc8f89d6742019648009ca6a11d18 is 50, key is test_row_0/C:col10/1731255235641/Put/seqid=0 2024-11-10T16:13:57,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742008_1184 (size=9757) 2024-11-10T16:13:57,931 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5b8dc8f89d6742019648009ca6a11d18 2024-11-10T16:13:57,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e2fada4f92684d4aa9dc510e0bd7b939 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939 2024-11-10T16:13:57,941 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939, entries=100, sequenceid=250, filesize=21.9 K 2024-11-10T16:13:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/5a57d099dbec425a91200e38bcbf8614 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614 2024-11-10T16:13:57,948 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614, entries=100, sequenceid=250, filesize=9.5 K 2024-11-10T16:13:57,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5b8dc8f89d6742019648009ca6a11d18 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18 2024-11-10T16:13:57,953 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18, entries=100, sequenceid=250, filesize=9.5 K 2024-11-10T16:13:57,954 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 54d3f36a627b699150cc523f346a55d2 in 1290ms, sequenceid=250, compaction requested=false 2024-11-10T16:13:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T16:13:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T16:13:57,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T16:13:57,957 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3680 sec 2024-11-10T16:13:57,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3740 sec 2024-11-10T16:13:57,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:57,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:57,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:13:57,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111015c25b94db3240dfbe723c52d905d227_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255297977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255297978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255297978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742009_1185 (size=14994) 2024-11-10T16:13:57,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255297980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:57,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255297980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:57,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255298083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255298093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255298093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255298103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255298104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255298293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255298303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255298303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255298313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255298313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:13:58,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111015c25b94db3240dfbe723c52d905d227_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015c25b94db3240dfbe723c52d905d227_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:13:58,396 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/3ff18467e4d140eb9148f70b243cd99b, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:58,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/3ff18467e4d140eb9148f70b243cd99b is 175, key is test_row_0/A:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:58,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742010_1186 (size=39949) 2024-11-10T16:13:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255298603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255298613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255298613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255298623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255298623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:58,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:58,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/3ff18467e4d140eb9148f70b243cd99b 2024-11-10T16:13:58,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/71b1341755fe44d38481d339c00a5d45 is 50, key is test_row_0/B:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742011_1187 (size=12301) 2024-11-10T16:13:59,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255299113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:59,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:59,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255299124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255299133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255299133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:59,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255299133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:13:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:13:59,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/71b1341755fe44d38481d339c00a5d45 2024-11-10T16:13:59,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4a44760c4ec744c789f36916afd73f32 is 50, key is test_row_0/C:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:59,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742012_1188 (size=12301) 2024-11-10T16:13:59,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4a44760c4ec744c789f36916afd73f32 2024-11-10T16:13:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/3ff18467e4d140eb9148f70b243cd99b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b 2024-11-10T16:13:59,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b, entries=200, sequenceid=272, filesize=39.0 K 2024-11-10T16:13:59,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/71b1341755fe44d38481d339c00a5d45 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45 2024-11-10T16:13:59,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45, entries=150, sequenceid=272, filesize=12.0 K 2024-11-10T16:13:59,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4a44760c4ec744c789f36916afd73f32 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32 2024-11-10T16:13:59,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32, entries=150, sequenceid=272, filesize=12.0 K 2024-11-10T16:13:59,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 54d3f36a627b699150cc523f346a55d2 in 1686ms, sequenceid=272, compaction requested=true 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:59,653 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:13:59,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:59,653 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:59,654 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:59,654 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:59,654 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:13:59,654 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:13:59,654 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:59,654 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:59,654 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d24aee01fb874f4b85bb68b065740879, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=33.8 K 2024-11-10T16:13:59,654 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=91.7 K 2024-11-10T16:13:59,654 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:59,655 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b] 2024-11-10T16:13:59,655 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d24aee01fb874f4b85bb68b065740879, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234990 2024-11-10T16:13:59,655 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4765b5eed5a84b8e977a10d0ea3f0cf8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234990 2024-11-10T16:13:59,656 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a57d099dbec425a91200e38bcbf8614, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731255235641 2024-11-10T16:13:59,656 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2fada4f92684d4aa9dc510e0bd7b939, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731255235641 2024-11-10T16:13:59,660 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 71b1341755fe44d38481d339c00a5d45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:13:59,660 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ff18467e4d140eb9148f70b243cd99b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:13:59,668 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:59,670 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110fc454b6d3326427ebbb3bcc515850661_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:59,672 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:59,672 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/9fdd9359dc03428fad3c370477fcfe6c is 50, key is test_row_0/B:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:59,672 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110fc454b6d3326427ebbb3bcc515850661_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:59,673 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fc454b6d3326427ebbb3bcc515850661_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:13:59,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742014_1190 (size=4469) 2024-11-10T16:13:59,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742013_1189 (size=12813) 2024-11-10T16:13:59,683 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#158 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:59,684 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f576bab3b406414583911d8ea7091a25 is 175, key is test_row_0/A:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:59,689 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/9fdd9359dc03428fad3c370477fcfe6c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/9fdd9359dc03428fad3c370477fcfe6c 2024-11-10T16:13:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742015_1191 (size=31767) 2024-11-10T16:13:59,698 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into 9fdd9359dc03428fad3c370477fcfe6c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:59,698 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:59,698 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255239653; duration=0sec 2024-11-10T16:13:59,698 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:13:59,698 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:13:59,699 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:13:59,699 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f576bab3b406414583911d8ea7091a25 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25 2024-11-10T16:13:59,700 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:13:59,700 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:13:59,700 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:59,700 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1c7aeba4a5eb4fa58312aea1fcd50a40, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=33.8 K 2024-11-10T16:13:59,701 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7aeba4a5eb4fa58312aea1fcd50a40, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731255234990 2024-11-10T16:13:59,701 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b8dc8f89d6742019648009ca6a11d18, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731255235641 2024-11-10T16:13:59,701 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a44760c4ec744c789f36916afd73f32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:13:59,705 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into f576bab3b406414583911d8ea7091a25(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:13:59,705 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:13:59,705 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255239653; duration=0sec 2024-11-10T16:13:59,705 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:13:59,705 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:13:59,713 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:13:59,714 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/55024d2110974d7e8ecab0409e04feeb is 50, key is test_row_0/C:col10/1731255236815/Put/seqid=0 2024-11-10T16:13:59,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742016_1192 (size=12779) 2024-11-10T16:13:59,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T16:13:59,753 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:13:59,753 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:13:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T16:13:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:13:59,755 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:13:59,756 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:13:59,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:13:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:13:59,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T16:13:59,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:13:59,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:13:59,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:13:59,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f08de5e2772b44059becfe8061353270_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255237977/Put/seqid=0 2024-11-10T16:13:59,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742017_1193 (size=12454) 2024-11-10T16:14:00,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:14:00,127 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/55024d2110974d7e8ecab0409e04feeb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/55024d2110974d7e8ecab0409e04feeb 2024-11-10T16:14:00,135 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 55024d2110974d7e8ecab0409e04feeb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:00,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:00,135 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=13, startTime=1731255239653; duration=0sec 2024-11-10T16:14:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:00,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:00,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:14:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255300156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255300158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255300162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731255240150, max timestamp=1731255240161Num uniques 859; 2024-11-10T16:14:00,165 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T16:14:00,170 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731255240150, max timestamp=1731255240161Num uniques 859; 2024-11-10T16:14:00,171 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T16:14:00,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731255240150, max timestamp=1731255240161Num uniques 859; 2024-11-10T16:14:00,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T16:14:00,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255300211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255300213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255300273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255300273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255300273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:00,327 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f08de5e2772b44059becfe8061353270_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f08de5e2772b44059becfe8061353270_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:00,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fc2077970a604ae88cb17a4911440465, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:00,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fc2077970a604ae88cb17a4911440465 is 175, key is test_row_0/A:col10/1731255237977/Put/seqid=0 2024-11-10T16:14:00,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255300323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255300323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742018_1194 (size=31255) 2024-11-10T16:14:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:14:00,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255300483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255300483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255300483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255300543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255300543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fc2077970a604ae88cb17a4911440465 2024-11-10T16:14:00,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/891bd83309c94c9b9404ad241a92ca96 is 50, key is test_row_0/B:col10/1731255237977/Put/seqid=0 2024-11-10T16:14:00,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742019_1195 (size=12301) 2024-11-10T16:14:00,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255300793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255300793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255300793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255300853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255300853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:00,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:14:01,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/891bd83309c94c9b9404ad241a92ca96 2024-11-10T16:14:01,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5406b5dbc08b4741805847bc1af49402 is 50, key is test_row_0/C:col10/1731255237977/Put/seqid=0 2024-11-10T16:14:01,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742020_1196 (size=12301) 2024-11-10T16:14:01,164 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5406b5dbc08b4741805847bc1af49402 2024-11-10T16:14:01,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fc2077970a604ae88cb17a4911440465 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465 2024-11-10T16:14:01,173 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465, entries=150, sequenceid=286, filesize=30.5 K 2024-11-10T16:14:01,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/891bd83309c94c9b9404ad241a92ca96 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96 2024-11-10T16:14:01,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T16:14:01,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5406b5dbc08b4741805847bc1af49402 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402 2024-11-10T16:14:01,183 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402, entries=150, sequenceid=286, filesize=12.0 K 2024-11-10T16:14:01,184 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 54d3f36a627b699150cc523f346a55d2 in 1275ms, sequenceid=286, compaction requested=false 2024-11-10T16:14:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:01,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T16:14:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T16:14:01,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T16:14:01,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-10T16:14:01,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4350 sec 2024-11-10T16:14:01,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:01,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:01,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:01,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b44907a6278d48f3aed93df64fb8026b_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742021_1197 (size=14994) 2024-11-10T16:14:01,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255301318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255301325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255301326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255301363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255301364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255301433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255301434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255301434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255301643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255301643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255301643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,724 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:01,728 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b44907a6278d48f3aed93df64fb8026b_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b44907a6278d48f3aed93df64fb8026b_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:01,729 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1cd5480290fa409b8ffc75f7b6ca3501, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1cd5480290fa409b8ffc75f7b6ca3501 is 175, key is test_row_0/A:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742022_1198 (size=39949) 2024-11-10T16:14:01,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1cd5480290fa409b8ffc75f7b6ca3501 2024-11-10T16:14:01,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/7f020efbdeda4da3968f535c24357fa3 is 50, key is test_row_0/B:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742023_1199 (size=12301) 2024-11-10T16:14:01,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/7f020efbdeda4da3968f535c24357fa3 2024-11-10T16:14:01,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/01f393e09eac45f29477dd3c098ed9b0 is 50, key is test_row_0/C:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742024_1200 (size=12301) 2024-11-10T16:14:01,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/01f393e09eac45f29477dd3c098ed9b0 2024-11-10T16:14:01,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1cd5480290fa409b8ffc75f7b6ca3501 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501 2024-11-10T16:14:01,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501, entries=200, sequenceid=321, filesize=39.0 K 2024-11-10T16:14:01,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/7f020efbdeda4da3968f535c24357fa3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3 2024-11-10T16:14:01,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T16:14:01,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/01f393e09eac45f29477dd3c098ed9b0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0 2024-11-10T16:14:01,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T16:14:01,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 54d3f36a627b699150cc523f346a55d2 in 480ms, sequenceid=321, compaction requested=true 2024-11-10T16:14:01,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:01,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:01,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:01,788 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:01,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:01,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:01,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:01,789 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:14:01,789 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:01,790 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:01,790 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:14:01,790 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:01,790 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=100.6 K 2024-11-10T16:14:01,790 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:01,790 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501] 2024-11-10T16:14:01,792 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:01,792 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:14:01,792 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f576bab3b406414583911d8ea7091a25, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:14:01,792 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:01,792 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/9fdd9359dc03428fad3c370477fcfe6c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.5 K 2024-11-10T16:14:01,792 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc2077970a604ae88cb17a4911440465, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731255237976 2024-11-10T16:14:01,793 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd5480290fa409b8ffc75f7b6ca3501, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:01,793 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fdd9359dc03428fad3c370477fcfe6c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:14:01,793 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 891bd83309c94c9b9404ad241a92ca96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731255237976 2024-11-10T16:14:01,794 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f020efbdeda4da3968f535c24357fa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:01,802 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,804 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:01,805 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/7a70bf470fa64459867e8e8d3639a848 is 50, key is test_row_0/B:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,806 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110713928b3a49c4864a5e97b8daf5e357c_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,808 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110713928b3a49c4864a5e97b8daf5e357c_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,808 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110713928b3a49c4864a5e97b8daf5e357c_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742025_1201 (size=12915) 2024-11-10T16:14:01,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742026_1202 (size=4469) 2024-11-10T16:14:01,821 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/7a70bf470fa64459867e8e8d3639a848 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7a70bf470fa64459867e8e8d3639a848 2024-11-10T16:14:01,826 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into 7a70bf470fa64459867e8e8d3639a848(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:01,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:01,826 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255241788; duration=0sec 2024-11-10T16:14:01,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:01,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:14:01,826 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:01,827 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:01,828 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:14:01,828 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:01,828 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/55024d2110974d7e8ecab0409e04feeb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.5 K 2024-11-10T16:14:01,828 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 55024d2110974d7e8ecab0409e04feeb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731255236815 2024-11-10T16:14:01,829 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5406b5dbc08b4741805847bc1af49402, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731255237976 2024-11-10T16:14:01,829 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f393e09eac45f29477dd3c098ed9b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:01,837 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:01,837 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/6a382e812ca74c239e176a158b3d1d2d is 50, key is test_row_0/C:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742027_1203 (size=12881) 2024-11-10T16:14:01,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T16:14:01,903 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:01,903 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:01,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T16:14:01,905 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:01,906 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:01,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:01,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:01,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:01,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:01,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f1bd9ec4b9b3450c83a5ed0567f5b241_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255241954/Put/seqid=0 2024-11-10T16:14:01,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742028_1204 (size=17534) 2024-11-10T16:14:01,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:01,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f1bd9ec4b9b3450c83a5ed0567f5b241_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f1bd9ec4b9b3450c83a5ed0567f5b241_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:01,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b9e2c2ad35894062b911ff04e0ea78d8, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:01,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b9e2c2ad35894062b911ff04e0ea78d8 is 175, key is test_row_0/A:col10/1731255241954/Put/seqid=0 2024-11-10T16:14:01,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742029_1205 (size=48639) 2024-11-10T16:14:01,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255301984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255301986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255301988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:02,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255302103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255302103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255302103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,216 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#167 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:02,217 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0e4ac58e85424f4fbf1860e2dd2ad5d6 is 175, key is test_row_0/A:col10/1731255240181/Put/seqid=0 2024-11-10T16:14:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:02,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742030_1206 (size=31869) 2024-11-10T16:14:02,231 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/0e4ac58e85424f4fbf1860e2dd2ad5d6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6 2024-11-10T16:14:02,237 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into 0e4ac58e85424f4fbf1860e2dd2ad5d6(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:02,237 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:02,237 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255241788; duration=0sec 2024-11-10T16:14:02,237 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:02,237 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:14:02,247 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/6a382e812ca74c239e176a158b3d1d2d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/6a382e812ca74c239e176a158b3d1d2d 2024-11-10T16:14:02,255 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 6a382e812ca74c239e176a158b3d1d2d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:02,255 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:02,255 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=13, startTime=1731255241788; duration=0sec 2024-11-10T16:14:02,256 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:02,256 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:14:02,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255302313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255302313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255302313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255302383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255302383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b9e2c2ad35894062b911ff04e0ea78d8 2024-11-10T16:14:02,394 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,394 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/daa13b78b134437a978642eb87de3c3d is 50, key is test_row_0/B:col10/1731255241954/Put/seqid=0 2024-11-10T16:14:02,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742031_1207 (size=12301) 2024-11-10T16:14:02,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:02,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255302623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255302623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255302623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:02,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/daa13b78b134437a978642eb87de3c3d 2024-11-10T16:14:02,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5f149a34e14146bdbef7d059f75c93fb is 50, key is test_row_0/C:col10/1731255241954/Put/seqid=0 2024-11-10T16:14:02,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742032_1208 (size=12301) 2024-11-10T16:14:02,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:02,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:02,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:03,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:03,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:03,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:03,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:03,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:03,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255303133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255303133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255303133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:03,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5f149a34e14146bdbef7d059f75c93fb 2024-11-10T16:14:03,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b9e2c2ad35894062b911ff04e0ea78d8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8 2024-11-10T16:14:03,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8, entries=250, sequenceid=336, filesize=47.5 K 2024-11-10T16:14:03,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/daa13b78b134437a978642eb87de3c3d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d 2024-11-10T16:14:03,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d, entries=150, sequenceid=336, filesize=12.0 K 2024-11-10T16:14:03,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/5f149a34e14146bdbef7d059f75c93fb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb 2024-11-10T16:14:03,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb, entries=150, sequenceid=336, filesize=12.0 K 2024-11-10T16:14:03,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 54d3f36a627b699150cc523f346a55d2 in 1291ms, sequenceid=336, compaction requested=false 2024-11-10T16:14:03,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:03,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T16:14:03,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:03,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:03,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e930ddf5306349b6b01b31050db5df18_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255241984/Put/seqid=0 2024-11-10T16:14:03,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742033_1209 (size=12454) 2024-11-10T16:14:03,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:03,307 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e930ddf5306349b6b01b31050db5df18_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e930ddf5306349b6b01b31050db5df18_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:03,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f406964aeb7d4e29b9bdd34505cb3861, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:03,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f406964aeb7d4e29b9bdd34505cb3861 is 175, key is test_row_0/A:col10/1731255241984/Put/seqid=0 2024-11-10T16:14:03,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742034_1210 (size=31255) 2024-11-10T16:14:03,686 INFO [master/8e3de9160f9f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T16:14:03,686 INFO [master/8e3de9160f9f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T16:14:03,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f406964aeb7d4e29b9bdd34505cb3861 2024-11-10T16:14:03,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2523c01cd6fb44449dcca2f7f78ce3e3 is 50, key is test_row_0/B:col10/1731255241984/Put/seqid=0 2024-11-10T16:14:03,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742035_1211 (size=12301) 2024-11-10T16:14:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:04,143 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2523c01cd6fb44449dcca2f7f78ce3e3 2024-11-10T16:14:04,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:04,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:04,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1e52a2ee66b6401ab7905d7bf63d33e1 is 50, key is test_row_0/C:col10/1731255241984/Put/seqid=0 2024-11-10T16:14:04,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742036_1212 (size=12301) 2024-11-10T16:14:04,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255304166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255304168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255304169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,181 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255304273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255304273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255304283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255304403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,406 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,407 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255304413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,415 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255304483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255304483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255304494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1e52a2ee66b6401ab7905d7bf63d33e1 2024-11-10T16:14:04,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/f406964aeb7d4e29b9bdd34505cb3861 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861 2024-11-10T16:14:04,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861, entries=150, sequenceid=356, filesize=30.5 K 2024-11-10T16:14:04,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/2523c01cd6fb44449dcca2f7f78ce3e3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3 2024-11-10T16:14:04,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T16:14:04,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1e52a2ee66b6401ab7905d7bf63d33e1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1 2024-11-10T16:14:04,585 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1, entries=150, sequenceid=356, filesize=12.0 K 2024-11-10T16:14:04,587 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 54d3f36a627b699150cc523f346a55d2 in 1298ms, sequenceid=356, compaction requested=true 2024-11-10T16:14:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:04,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T16:14:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T16:14:04,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T16:14:04,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6820 sec 2024-11-10T16:14:04,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6870 sec 2024-11-10T16:14:04,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:04,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:04,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb06bcac80cd426fbf08d5646e70724a_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:04,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742037_1213 (size=17534) 2024-11-10T16:14:04,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255304829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255304830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255304831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255304943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255304944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255304944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255305153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255305153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255305153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:05,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110eb06bcac80cd426fbf08d5646e70724a_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06bcac80cd426fbf08d5646e70724a_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:05,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1e24fca8301040b6b8d66b0210a79dd3, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:05,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1e24fca8301040b6b8d66b0210a79dd3 is 175, key is test_row_0/A:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:05,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742038_1214 (size=48639) 2024-11-10T16:14:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255305463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255305463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255305463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,634 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=372, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1e24fca8301040b6b8d66b0210a79dd3 2024-11-10T16:14:05,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 is 50, key is test_row_0/B:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:05,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742039_1215 (size=12301) 2024-11-10T16:14:05,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255305974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255305974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255305974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:05,975 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:05,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:06,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 2024-11-10T16:14:06,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/de0e17737bb94ed69830ec095fadc811 is 50, key is test_row_0/C:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:06,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742040_1216 (size=12301) 2024-11-10T16:14:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T16:14:06,073 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:06,073 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T16:14:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:06,075 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:06,076 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:06,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:06,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T16:14:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T16:14:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:06,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/de0e17737bb94ed69830ec095fadc811 2024-11-10T16:14:06,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/1e24fca8301040b6b8d66b0210a79dd3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3 2024-11-10T16:14:06,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3, entries=250, sequenceid=372, filesize=47.5 K 2024-11-10T16:14:06,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 2024-11-10T16:14:06,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72, entries=150, sequenceid=372, filesize=12.0 K 2024-11-10T16:14:06,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/de0e17737bb94ed69830ec095fadc811 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811 2024-11-10T16:14:06,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811, entries=150, sequenceid=372, filesize=12.0 K 2024-11-10T16:14:06,485 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 54d3f36a627b699150cc523f346a55d2 in 1690ms, sequenceid=372, compaction requested=true 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:06,485 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:06,485 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:06,486 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:14:06,486 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:14:06,487 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,487 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7a70bf470fa64459867e8e8d3639a848, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=48.7 K 2024-11-10T16:14:06,487 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160402 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:14:06,487 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:14:06,487 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,487 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=156.6 K 2024-11-10T16:14:06,487 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,487 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3] 2024-11-10T16:14:06,487 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a70bf470fa64459867e8e8d3639a848, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:06,488 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4ac58e85424f4fbf1860e2dd2ad5d6, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:06,488 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting daa13b78b134437a978642eb87de3c3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731255241954 2024-11-10T16:14:06,488 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2523c01cd6fb44449dcca2f7f78ce3e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731255241977 2024-11-10T16:14:06,488 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e2c2ad35894062b911ff04e0ea78d8, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731255241310 2024-11-10T16:14:06,489 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f406964aeb7d4e29b9bdd34505cb3861, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731255241977 2024-11-10T16:14:06,489 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fed2f9f0d4dd4deeaa1baa0e7e539e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:06,489 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e24fca8301040b6b8d66b0210a79dd3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:06,498 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:06,499 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#179 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:06,500 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d782dc3a04d84badadd96ea94cfaeb05 is 50, key is test_row_0/B:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:06,503 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111032c257216dd04f209528a68764fc4a1b_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:06,506 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111032c257216dd04f209528a68764fc4a1b_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:06,506 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111032c257216dd04f209528a68764fc4a1b_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:06,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742041_1217 (size=13051) 2024-11-10T16:14:06,515 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/d782dc3a04d84badadd96ea94cfaeb05 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d782dc3a04d84badadd96ea94cfaeb05 2024-11-10T16:14:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742042_1218 (size=4469) 2024-11-10T16:14:06,521 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into d782dc3a04d84badadd96ea94cfaeb05(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:06,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:06,521 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=12, startTime=1731255246485; duration=0sec 2024-11-10T16:14:06,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:06,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:14:06,521 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:14:06,522 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:14:06,523 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:14:06,523 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,523 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/6a382e812ca74c239e176a158b3d1d2d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=48.6 K 2024-11-10T16:14:06,523 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a382e812ca74c239e176a158b3d1d2d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255240181 2024-11-10T16:14:06,524 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f149a34e14146bdbef7d059f75c93fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731255241954 2024-11-10T16:14:06,524 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e52a2ee66b6401ab7905d7bf63d33e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731255241977 2024-11-10T16:14:06,524 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting de0e17737bb94ed69830ec095fadc811, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:06,533 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:06,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T16:14:06,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:06,536 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/0bdc4d3b9d7a48638ba7d2e42837d339 is 50, key is test_row_0/C:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:06,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:06,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111025e32c1232404ef797724da317a1e1dd_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255244830/Put/seqid=0 2024-11-10T16:14:06,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742043_1219 (size=13017) 2024-11-10T16:14:06,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742044_1220 (size=12454) 2024-11-10T16:14:06,555 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/0bdc4d3b9d7a48638ba7d2e42837d339 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0bdc4d3b9d7a48638ba7d2e42837d339 2024-11-10T16:14:06,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:06,560 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111025e32c1232404ef797724da317a1e1dd_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111025e32c1232404ef797724da317a1e1dd_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:06,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/6d7b8e6c119a4e1083a9ba5729d2860c, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:06,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/6d7b8e6c119a4e1083a9ba5729d2860c is 175, key is test_row_0/A:col10/1731255244830/Put/seqid=0 2024-11-10T16:14:06,562 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 0bdc4d3b9d7a48638ba7d2e42837d339(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:06,562 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:06,562 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=12, startTime=1731255246485; duration=0sec 2024-11-10T16:14:06,562 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:06,562 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:14:06,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742045_1221 (size=31255) 2024-11-10T16:14:06,566 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/6d7b8e6c119a4e1083a9ba5729d2860c 2024-11-10T16:14:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/548d33e5b2f344968affcb8b80c9868a is 50, key is test_row_0/B:col10/1731255244830/Put/seqid=0 2024-11-10T16:14:06,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742046_1222 (size=12301) 2024-11-10T16:14:06,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:06,916 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#180 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:06,917 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/52900b4c7bf448099a405a77f2369dc9 is 175, key is test_row_0/A:col10/1731255244795/Put/seqid=0 2024-11-10T16:14:06,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742047_1223 (size=32005) 2024-11-10T16:14:06,977 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/548d33e5b2f344968affcb8b80c9868a 2024-11-10T16:14:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:06,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:06,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4323aa27849b41a5b28772ad585c21b6 is 50, key is test_row_0/C:col10/1731255244830/Put/seqid=0 2024-11-10T16:14:06,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742048_1224 (size=12301) 2024-11-10T16:14:07,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:07,327 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/52900b4c7bf448099a405a77f2369dc9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9 2024-11-10T16:14:07,332 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into 52900b4c7bf448099a405a77f2369dc9(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:07,332 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:07,332 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=12, startTime=1731255246485; duration=0sec 2024-11-10T16:14:07,332 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:07,332 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:14:07,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,392 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4323aa27849b41a5b28772ad585c21b6 2024-11-10T16:14:07,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/6d7b8e6c119a4e1083a9ba5729d2860c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c 2024-11-10T16:14:07,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c, entries=150, sequenceid=389, filesize=30.5 K 2024-11-10T16:14:07,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/548d33e5b2f344968affcb8b80c9868a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a 2024-11-10T16:14:07,407 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a, entries=150, sequenceid=389, filesize=12.0 K 2024-11-10T16:14:07,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/4323aa27849b41a5b28772ad585c21b6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6 2024-11-10T16:14:07,412 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6, entries=150, sequenceid=389, filesize=12.0 K 2024-11-10T16:14:07,412 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 54d3f36a627b699150cc523f346a55d2 in 877ms, sequenceid=389, compaction requested=false 2024-11-10T16:14:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:07,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T16:14:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T16:14:07,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T16:14:07,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-11-10T16:14:07,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3430 sec 2024-11-10T16:14:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:07,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:07,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:07,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba09a9bc636b4b6f9e41052e35205916_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:07,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742049_1225 (size=17534) 2024-11-10T16:14:07,664 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:07,668 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba09a9bc636b4b6f9e41052e35205916_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba09a9bc636b4b6f9e41052e35205916_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:07,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/bad38d161b5643e98df9348c944b116e, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:07,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/bad38d161b5643e98df9348c944b116e is 175, key is test_row_0/A:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742050_1226 (size=48639) 2024-11-10T16:14:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255307983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255307993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255307993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:07,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,074 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/bad38d161b5643e98df9348c944b116e 2024-11-10T16:14:08,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/6e38b68234144e808c5cce1a679d571f is 50, key is test_row_0/B:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:08,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742051_1227 (size=12301) 2024-11-10T16:14:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T16:14:08,224 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:08,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:08,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-10T16:14:08,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:08,226 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:08,227 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:08,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:08,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255308293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255308303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255308303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:08,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T16:14:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52088 deadline: 1731255308423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,427 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:08,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52096 deadline: 1731255308433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,440 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 54d3f36a627b699150cc523f346a55d2 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/6e38b68234144e808c5cce1a679d571f 2024-11-10T16:14:08,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1f9434be652744a9829a96b79cdc687b is 50, key is test_row_0/C:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:08,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742052_1228 (size=12301) 2024-11-10T16:14:08,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T16:14:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:08,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T16:14:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:08,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255308803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255308813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255308813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:08,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:08,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T16:14:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:08,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1f9434be652744a9829a96b79cdc687b 2024-11-10T16:14:08,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/bad38d161b5643e98df9348c944b116e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e 2024-11-10T16:14:08,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e, entries=250, sequenceid=409, filesize=47.5 K 2024-11-10T16:14:08,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/6e38b68234144e808c5cce1a679d571f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f 2024-11-10T16:14:08,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T16:14:08,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1f9434be652744a9829a96b79cdc687b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b 2024-11-10T16:14:08,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b, entries=150, sequenceid=409, filesize=12.0 K 2024-11-10T16:14:08,922 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 54d3f36a627b699150cc523f346a55d2 in 1275ms, sequenceid=409, compaction requested=true 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:08,922 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:08,922 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:08,923 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:08,923 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:08,923 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:14:08,923 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:14:08,924 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,924 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,924 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d782dc3a04d84badadd96ea94cfaeb05, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.8 K 2024-11-10T16:14:08,924 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=109.3 K 2024-11-10T16:14:08,924 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,924 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e] 2024-11-10T16:14:08,924 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d782dc3a04d84badadd96ea94cfaeb05, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:08,924 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52900b4c7bf448099a405a77f2369dc9, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:08,925 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 548d33e5b2f344968affcb8b80c9868a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731255244805 2024-11-10T16:14:08,925 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7b8e6c119a4e1083a9ba5729d2860c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731255244805 2024-11-10T16:14:08,925 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting bad38d161b5643e98df9348c944b116e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:08,925 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e38b68234144e808c5cce1a679d571f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:08,932 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:08,934 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:08,934 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/45c84a556319433ca0b53c9980926757 is 50, key is test_row_0/B:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:08,937 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111021c5736ff34b423aa9116302be5f9ba2_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:08,939 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111021c5736ff34b423aa9116302be5f9ba2_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:08,939 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111021c5736ff34b423aa9116302be5f9ba2_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:08,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742053_1229 (size=13153) 2024-11-10T16:14:08,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742054_1230 (size=4469) 2024-11-10T16:14:08,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:08,992 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:08,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e32c703742048d796012d8eb0fc8bac_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255247668/Put/seqid=0 2024-11-10T16:14:09,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742055_1231 (size=12454) 2024-11-10T16:14:09,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:09,008 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e32c703742048d796012d8eb0fc8bac_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e32c703742048d796012d8eb0fc8bac_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/44e4ba95934a43e0b8b66ca0656f3f76, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/44e4ba95934a43e0b8b66ca0656f3f76 is 175, key is test_row_0/A:col10/1731255247668/Put/seqid=0 2024-11-10T16:14:09,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742056_1232 (size=31255) 2024-11-10T16:14:09,350 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#188 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:09,351 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e6f29a6e72fb4c08872ebcb759322aeb is 175, key is test_row_0/A:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:09,352 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/45c84a556319433ca0b53c9980926757 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/45c84a556319433ca0b53c9980926757 2024-11-10T16:14:09,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742057_1233 (size=32107) 2024-11-10T16:14:09,360 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into 45c84a556319433ca0b53c9980926757(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:09,360 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:09,360 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255248922; duration=0sec 2024-11-10T16:14:09,360 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:09,360 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:14:09,360 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:09,362 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:09,362 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/e6f29a6e72fb4c08872ebcb759322aeb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb 2024-11-10T16:14:09,362 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:14:09,362 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:09,362 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0bdc4d3b9d7a48638ba7d2e42837d339, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.7 K 2024-11-10T16:14:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:09,363 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bdc4d3b9d7a48638ba7d2e42837d339, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731255244163 2024-11-10T16:14:09,364 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4323aa27849b41a5b28772ad585c21b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731255244805 2024-11-10T16:14:09,364 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9434be652744a9829a96b79cdc687b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:09,367 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into e6f29a6e72fb4c08872ebcb759322aeb(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:09,367 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:09,367 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255248922; duration=0sec 2024-11-10T16:14:09,368 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:09,368 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:14:09,372 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:09,372 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1a73f890ce444907a5c231e41b2bdb6d is 50, key is test_row_0/C:col10/1731255246999/Put/seqid=0 2024-11-10T16:14:09,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742058_1234 (size=13119) 2024-11-10T16:14:09,383 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/1a73f890ce444907a5c231e41b2bdb6d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1a73f890ce444907a5c231e41b2bdb6d 2024-11-10T16:14:09,388 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into 1a73f890ce444907a5c231e41b2bdb6d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:09,388 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:09,388 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=13, startTime=1731255248922; duration=0sec 2024-11-10T16:14:09,388 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:09,389 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:14:09,414 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/44e4ba95934a43e0b8b66ca0656f3f76 2024-11-10T16:14:09,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/10f3a71db6764bd9b2944d67a93a443a is 50, key is test_row_0/B:col10/1731255247668/Put/seqid=0 2024-11-10T16:14:09,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742059_1235 (size=12301) 2024-11-10T16:14:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:09,827 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/10f3a71db6764bd9b2944d67a93a443a 2024-11-10T16:14:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc is 50, key is test_row_0/C:col10/1731255247668/Put/seqid=0 2024-11-10T16:14:09,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742060_1236 (size=12301) 2024-11-10T16:14:09,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc 2024-11-10T16:14:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255309842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:09,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/44e4ba95934a43e0b8b66ca0656f3f76 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76 2024-11-10T16:14:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255309845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255309846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:09,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76, entries=150, sequenceid=424, filesize=30.5 K 2024-11-10T16:14:09,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/10f3a71db6764bd9b2944d67a93a443a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a 2024-11-10T16:14:09,860 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T16:14:09,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc 2024-11-10T16:14:09,865 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc, entries=150, sequenceid=424, filesize=12.0 K 2024-11-10T16:14:09,867 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 54d3f36a627b699150cc523f346a55d2 in 874ms, sequenceid=424, compaction requested=false 2024-11-10T16:14:09,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:09,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:09,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-10T16:14:09,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-10T16:14:09,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T16:14:09,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-10T16:14:09,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6470 sec 2024-11-10T16:14:09,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:09,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:09,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110307bf7fe96554cae9ed22d1ed8e14038_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742061_1237 (size=17534) 2024-11-10T16:14:09,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255309975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255309979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255309980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255310083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255310093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255310094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255310293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,297 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255310303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255310303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,371 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T16:14:10,373 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:10,374 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-10T16:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:10,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:10,377 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:10,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:10,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110307bf7fe96554cae9ed22d1ed8e14038_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110307bf7fe96554cae9ed22d1ed8e14038_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:10,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fcb4ff9ce8e54ddab3b83e4f726b7c15, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:10,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 is 175, key is test_row_0/A:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:10,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742062_1238 (size=48639) 2024-11-10T16:14:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:10,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:10,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:10,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255310603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,606 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,607 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255310613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255310613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:10,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:10,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=446, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 2024-11-10T16:14:10,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c817e390220f42929092dc6d2845694a is 50, key is test_row_0/B:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:10,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742063_1239 (size=12301) 2024-11-10T16:14:10,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c817e390220f42929092dc6d2845694a 2024-11-10T16:14:10,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/fd776144292845b3b14b5c5ec3836e9c is 50, key is test_row_0/C:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:10,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742064_1240 (size=12301) 2024-11-10T16:14:10,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:10,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:10,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:10,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:10,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:11,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:11,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52016 deadline: 1731255311113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:11,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52032 deadline: 1731255311123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:11,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:11,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52060 deadline: 1731255311133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:11,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:11,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:11,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:11,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:11,152 INFO [Thread-658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-658. 2024-11-10T16:14:11,152 DEBUG [Thread-658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:11,152 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,152 DEBUG [Thread-658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:11,153 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-10T16:14:11,153 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:11,153 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,153 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:11,155 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-10T16:14:11,155 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:11,155 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,156 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:11,158 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-10T16:14:11,158 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:11,158 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,158 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:11,158 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:11,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/fd776144292845b3b14b5c5ec3836e9c 2024-11-10T16:14:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 2024-11-10T16:14:11,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15, entries=250, sequenceid=446, filesize=47.5 K 2024-11-10T16:14:11,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/c817e390220f42929092dc6d2845694a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a 2024-11-10T16:14:11,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T16:14:11,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/fd776144292845b3b14b5c5ec3836e9c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c 2024-11-10T16:14:11,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T16:14:11,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 54d3f36a627b699150cc523f346a55d2 in 1274ms, sequenceid=446, compaction requested=true 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:11,230 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54d3f36a627b699150cc523f346a55d2:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:11,230 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/A is initiating minor compaction (all files) 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/B is initiating minor compaction (all files) 2024-11-10T16:14:11,231 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/A in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,231 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/B in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,231 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=109.4 K 2024-11-10T16:14:11,231 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/45c84a556319433ca0b53c9980926757, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.9 K 2024-11-10T16:14:11,231 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15] 2024-11-10T16:14:11,231 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 45c84a556319433ca0b53c9980926757, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:11,232 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6f29a6e72fb4c08872ebcb759322aeb, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:11,232 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f3a71db6764bd9b2944d67a93a443a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255247661 2024-11-10T16:14:11,232 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44e4ba95934a43e0b8b66ca0656f3f76, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255247661 2024-11-10T16:14:11,232 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c817e390220f42929092dc6d2845694a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731255249841 2024-11-10T16:14:11,232 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb4ff9ce8e54ddab3b83e4f726b7c15, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731255249841 2024-11-10T16:14:11,241 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#B#compaction#197 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:11,242 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:11,242 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/573a45452b6a4723b09c5a46c0c108c9 is 50, key is test_row_0/B:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:11,244 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111062956567814c4363babe98e2e5d72cf4_54d3f36a627b699150cc523f346a55d2 store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:11,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742065_1241 (size=13255) 2024-11-10T16:14:11,262 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111062956567814c4363babe98e2e5d72cf4_54d3f36a627b699150cc523f346a55d2, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:11,262 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111062956567814c4363babe98e2e5d72cf4_54d3f36a627b699150cc523f346a55d2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:11,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742066_1242 (size=4469) 2024-11-10T16:14:11,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-10T16:14:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:14:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:11,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:11,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:11,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ed87539e2d0d4037a79fc4988a8e89a3_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255249977/Put/seqid=0 2024-11-10T16:14:11,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742067_1243 (size=12454) 2024-11-10T16:14:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:11,654 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/573a45452b6a4723b09c5a46c0c108c9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/573a45452b6a4723b09c5a46c0c108c9 2024-11-10T16:14:11,671 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/B of 54d3f36a627b699150cc523f346a55d2 into 573a45452b6a4723b09c5a46c0c108c9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:11,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:11,671 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/B, priority=13, startTime=1731255251230; duration=0sec 2024-11-10T16:14:11,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:11,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:B 2024-11-10T16:14:11,671 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#A#compaction#198 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:11,671 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:11,672 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b96d3be995c341369e1dadfccbcd4edd is 175, key is test_row_0/A:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:11,672 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:11,672 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 54d3f36a627b699150cc523f346a55d2/C is initiating minor compaction (all files) 2024-11-10T16:14:11,672 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54d3f36a627b699150cc523f346a55d2/C in TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:11,672 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1a73f890ce444907a5c231e41b2bdb6d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp, totalSize=36.8 K 2024-11-10T16:14:11,673 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a73f890ce444907a5c231e41b2bdb6d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1731255246999 2024-11-10T16:14:11,673 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bcd3aec51f54eb7891d2ffc5d7b4ecc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731255247661 2024-11-10T16:14:11,674 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fd776144292845b3b14b5c5ec3836e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1731255249841 2024-11-10T16:14:11,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742068_1244 (size=32209) 2024-11-10T16:14:11,682 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54d3f36a627b699150cc523f346a55d2#C#compaction#200 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:11,682 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/da175de54ab44bcb9feee22917b61297 is 50, key is test_row_0/C:col10/1731255249841/Put/seqid=0 2024-11-10T16:14:11,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742069_1245 (size=13221) 2024-11-10T16:14:11,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:11,712 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ed87539e2d0d4037a79fc4988a8e89a3_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed87539e2d0d4037a79fc4988a8e89a3_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/69be8fa10eb047d3b33c5d713cc7a510, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/69be8fa10eb047d3b33c5d713cc7a510 is 175, key is test_row_0/A:col10/1731255249977/Put/seqid=0 2024-11-10T16:14:11,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742070_1246 (size=31255) 2024-11-10T16:14:12,081 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/b96d3be995c341369e1dadfccbcd4edd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b96d3be995c341369e1dadfccbcd4edd 2024-11-10T16:14:12,085 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/A of 54d3f36a627b699150cc523f346a55d2 into b96d3be995c341369e1dadfccbcd4edd(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:12,085 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:12,085 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/A, priority=13, startTime=1731255251230; duration=0sec 2024-11-10T16:14:12,085 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:12,085 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:A 2024-11-10T16:14:12,089 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/da175de54ab44bcb9feee22917b61297 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/da175de54ab44bcb9feee22917b61297 2024-11-10T16:14:12,093 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54d3f36a627b699150cc523f346a55d2/C of 54d3f36a627b699150cc523f346a55d2 into da175de54ab44bcb9feee22917b61297(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:12,093 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:12,093 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., storeName=54d3f36a627b699150cc523f346a55d2/C, priority=13, startTime=1731255251230; duration=0sec 2024-11-10T16:14:12,094 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:12,094 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54d3f36a627b699150cc523f346a55d2:C 2024-11-10T16:14:12,117 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/69be8fa10eb047d3b33c5d713cc7a510 2024-11-10T16:14:12,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/1c4030648e9045d1b045493afdc57c34 is 50, key is test_row_0/B:col10/1731255249977/Put/seqid=0 2024-11-10T16:14:12,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742071_1247 (size=12301) 2024-11-10T16:14:12,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:12,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. as already flushing 2024-11-10T16:14:12,135 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-10T16:14:12,135 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:12,135 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,135 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:12,145 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-10T16:14:12,145 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-10T16:14:12,145 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:12,145 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:12,145 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,145 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,145 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,145 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:12,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:12,145 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:12,527 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/1c4030648e9045d1b045493afdc57c34 2024-11-10T16:14:12,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/dd9f92ef9bd6497b975caf78d1ae9040 is 50, key is test_row_0/C:col10/1731255249977/Put/seqid=0 2024-11-10T16:14:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742072_1248 (size=12301) 2024-11-10T16:14:12,938 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/dd9f92ef9bd6497b975caf78d1ae9040 2024-11-10T16:14:12,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/69be8fa10eb047d3b33c5d713cc7a510 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/69be8fa10eb047d3b33c5d713cc7a510 2024-11-10T16:14:12,946 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/69be8fa10eb047d3b33c5d713cc7a510, entries=150, sequenceid=459, filesize=30.5 K 2024-11-10T16:14:12,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/1c4030648e9045d1b045493afdc57c34 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/1c4030648e9045d1b045493afdc57c34 2024-11-10T16:14:12,951 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/1c4030648e9045d1b045493afdc57c34, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T16:14:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/dd9f92ef9bd6497b975caf78d1ae9040 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/dd9f92ef9bd6497b975caf78d1ae9040 2024-11-10T16:14:12,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/dd9f92ef9bd6497b975caf78d1ae9040, entries=150, sequenceid=459, filesize=12.0 K 2024-11-10T16:14:12,956 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 54d3f36a627b699150cc523f346a55d2 in 1659ms, sequenceid=459, compaction requested=false 2024-11-10T16:14:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 54d3f36a627b699150cc523f346a55d2: 2024-11-10T16:14:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:12,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-10T16:14:12,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-10T16:14:12,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T16:14:12,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5800 sec 2024-11-10T16:14:12,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 2.5850 sec 2024-11-10T16:14:14,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-10T16:14:14,544 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:18,366 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T16:14:18,444 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-10T16:14:18,444 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:18,444 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,445 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:18,454 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-10T16:14:18,454 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:18,454 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,454 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,454 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2951 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3097 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1273 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3818 rows 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1275 2024-11-10T16:14:18,455 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3824 rows 2024-11-10T16:14:18,455 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:14:18,455 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:14:18,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:18,455 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:18,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:14:18,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:18,459 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255258459"}]},"ts":"1731255258459"} 2024-11-10T16:14:18,460 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:14:18,460 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:14:18,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:14:18,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, UNASSIGN}] 2024-11-10T16:14:18,463 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, UNASSIGN 2024-11-10T16:14:18,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:18,464 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41631 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=8e3de9160f9f,44105,1731255200149, table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-10T16:14:18,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, UNASSIGN because future has completed 2024-11-10T16:14:18,465 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:14:18,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:18,617 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(122): Close 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:18,617 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:14:18,617 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1722): Closing 54d3f36a627b699150cc523f346a55d2, disabling compactions & flushes 2024-11-10T16:14:18,617 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:18,617 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:18,617 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. after waiting 0 ms 2024-11-10T16:14:18,617 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:18,617 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(2902): Flushing 54d3f36a627b699150cc523f346a55d2 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=A 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=B 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 54d3f36a627b699150cc523f346a55d2, store=C 2024-11-10T16:14:18,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:18,624 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f17326153395448093e39fc0c4cbc4b8_54d3f36a627b699150cc523f346a55d2 is 50, key is test_row_0/A:col10/1731255252144/Put/seqid=0 2024-11-10T16:14:18,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742073_1249 (size=9914) 2024-11-10T16:14:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:19,034 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:19,038 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f17326153395448093e39fc0c4cbc4b8_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f17326153395448093e39fc0c4cbc4b8_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:19,039 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/37543ea0025643bcaa1bafee579a3999, store: [table=TestAcidGuarantees family=A region=54d3f36a627b699150cc523f346a55d2] 2024-11-10T16:14:19,040 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/37543ea0025643bcaa1bafee579a3999 is 175, key is test_row_0/A:col10/1731255252144/Put/seqid=0 2024-11-10T16:14:19,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742074_1250 (size=22561) 2024-11-10T16:14:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:19,444 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=470, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/37543ea0025643bcaa1bafee579a3999 2024-11-10T16:14:19,451 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a63fbba2236f4859a95d49fc5e11f0bf is 50, key is test_row_0/B:col10/1731255252144/Put/seqid=0 2024-11-10T16:14:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742075_1251 (size=9857) 2024-11-10T16:14:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:19,855 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a63fbba2236f4859a95d49fc5e11f0bf 2024-11-10T16:14:19,861 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9566557a0bd24d649eca6406c288ba97 is 50, key is test_row_0/C:col10/1731255252144/Put/seqid=0 2024-11-10T16:14:19,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742076_1252 (size=9857) 2024-11-10T16:14:20,265 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9566557a0bd24d649eca6406c288ba97 2024-11-10T16:14:20,270 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/A/37543ea0025643bcaa1bafee579a3999 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/37543ea0025643bcaa1bafee579a3999 2024-11-10T16:14:20,273 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/37543ea0025643bcaa1bafee579a3999, entries=100, sequenceid=470, filesize=22.0 K 2024-11-10T16:14:20,274 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/B/a63fbba2236f4859a95d49fc5e11f0bf as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a63fbba2236f4859a95d49fc5e11f0bf 2024-11-10T16:14:20,278 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a63fbba2236f4859a95d49fc5e11f0bf, entries=100, sequenceid=470, filesize=9.6 K 2024-11-10T16:14:20,278 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/.tmp/C/9566557a0bd24d649eca6406c288ba97 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9566557a0bd24d649eca6406c288ba97 2024-11-10T16:14:20,281 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9566557a0bd24d649eca6406c288ba97, entries=100, sequenceid=470, filesize=9.6 K 2024-11-10T16:14:20,282 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 54d3f36a627b699150cc523f346a55d2 in 1665ms, sequenceid=470, compaction requested=true 2024-11-10T16:14:20,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15] to archive 2024-11-10T16:14:20,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:20,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b4393ec29d8048b3b7766164d64ec285 2024-11-10T16:14:20,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/582d8abc0edb49da937f6f6a69165f47 2024-11-10T16:14:20,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c1daeef7ed6d425dbc61cf9341a1b4ab 2024-11-10T16:14:20,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/652616392aae4b7483491c153a4ad90c 2024-11-10T16:14:20,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f13406b383b14b8986d55af297235208 2024-11-10T16:14:20,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fdf289632da24ee48ac4b0f3a3a97fba 2024-11-10T16:14:20,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/c407973812c245b98810d5c02775025b 2024-11-10T16:14:20,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/a1288cc998c84f1d851e1c4da4e5bd83 2024-11-10T16:14:20,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b766f413ebc45348c9edd176cd3ab64 2024-11-10T16:14:20,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/693d3294764b4f088f6be8d3419a6b16 2024-11-10T16:14:20,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0b59080acf24470392afdc11b8fab4c8 2024-11-10T16:14:20,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/80034c09d5904413bee800f5d6b87ceb 2024-11-10T16:14:20,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/4765b5eed5a84b8e977a10d0ea3f0cf8 2024-11-10T16:14:20,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e2fada4f92684d4aa9dc510e0bd7b939 2024-11-10T16:14:20,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/3ff18467e4d140eb9148f70b243cd99b 2024-11-10T16:14:20,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f576bab3b406414583911d8ea7091a25 2024-11-10T16:14:20,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fc2077970a604ae88cb17a4911440465 2024-11-10T16:14:20,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1cd5480290fa409b8ffc75f7b6ca3501 2024-11-10T16:14:20,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/0e4ac58e85424f4fbf1860e2dd2ad5d6 2024-11-10T16:14:20,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b9e2c2ad35894062b911ff04e0ea78d8 2024-11-10T16:14:20,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/f406964aeb7d4e29b9bdd34505cb3861 2024-11-10T16:14:20,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/1e24fca8301040b6b8d66b0210a79dd3 2024-11-10T16:14:20,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/52900b4c7bf448099a405a77f2369dc9 2024-11-10T16:14:20,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/6d7b8e6c119a4e1083a9ba5729d2860c 2024-11-10T16:14:20,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/bad38d161b5643e98df9348c944b116e 2024-11-10T16:14:20,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/e6f29a6e72fb4c08872ebcb759322aeb 2024-11-10T16:14:20,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/44e4ba95934a43e0b8b66ca0656f3f76 2024-11-10T16:14:20,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/fcb4ff9ce8e54ddab3b83e4f726b7c15 2024-11-10T16:14:20,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/ffa22081990a4ef381969f4dc3242fe4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/75247361850f4c1395ede38c32432131, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d01ab44dc0ef4873b1e1f57e66fc0d49, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d24aee01fb874f4b85bb68b065740879, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/9fdd9359dc03428fad3c370477fcfe6c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7a70bf470fa64459867e8e8d3639a848, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d782dc3a04d84badadd96ea94cfaeb05, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/45c84a556319433ca0b53c9980926757, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a] to archive 2024-11-10T16:14:20,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:20,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/e55684ea241148ecb60f6a30c06ad1ac 2024-11-10T16:14:20,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a2b80f57e1dc4aefbd2ee4d02199b723 2024-11-10T16:14:20,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/ffa22081990a4ef381969f4dc3242fe4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/ffa22081990a4ef381969f4dc3242fe4 2024-11-10T16:14:20,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/473482320aca4cb8bbcb800dcbd87eb3 2024-11-10T16:14:20,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/496b46cf84f040d5968f55a30f2ad8d1 2024-11-10T16:14:20,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/75247361850f4c1395ede38c32432131 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/75247361850f4c1395ede38c32432131 2024-11-10T16:14:20,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/f8ef7ef05d3946ebb7f905f97160e9ca 2024-11-10T16:14:20,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2af72f7443e84d0996a95b8aa9d522d8 2024-11-10T16:14:20,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d01ab44dc0ef4873b1e1f57e66fc0d49 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d01ab44dc0ef4873b1e1f57e66fc0d49 2024-11-10T16:14:20,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/92f38bfd9b0c465ba1804af09369ec7e 2024-11-10T16:14:20,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/66cbb7dfde82453f909733d1cbe1f342 2024-11-10T16:14:20,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d24aee01fb874f4b85bb68b065740879 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d24aee01fb874f4b85bb68b065740879 2024-11-10T16:14:20,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c2aa365acb5741e39f6c3b02e055dd6d 2024-11-10T16:14:20,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/5a57d099dbec425a91200e38bcbf8614 2024-11-10T16:14:20,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/9fdd9359dc03428fad3c370477fcfe6c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/9fdd9359dc03428fad3c370477fcfe6c 2024-11-10T16:14:20,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/71b1341755fe44d38481d339c00a5d45 2024-11-10T16:14:20,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/891bd83309c94c9b9404ad241a92ca96 2024-11-10T16:14:20,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7a70bf470fa64459867e8e8d3639a848 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7a70bf470fa64459867e8e8d3639a848 2024-11-10T16:14:20,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/7f020efbdeda4da3968f535c24357fa3 2024-11-10T16:14:20,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/daa13b78b134437a978642eb87de3c3d 2024-11-10T16:14:20,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/2523c01cd6fb44449dcca2f7f78ce3e3 2024-11-10T16:14:20,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d782dc3a04d84badadd96ea94cfaeb05 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/d782dc3a04d84badadd96ea94cfaeb05 2024-11-10T16:14:20,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/fed2f9f0d4dd4deeaa1baa0e7e539e72 2024-11-10T16:14:20,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/548d33e5b2f344968affcb8b80c9868a 2024-11-10T16:14:20,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/45c84a556319433ca0b53c9980926757 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/45c84a556319433ca0b53c9980926757 2024-11-10T16:14:20,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/6e38b68234144e808c5cce1a679d571f 2024-11-10T16:14:20,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/10f3a71db6764bd9b2944d67a93a443a 2024-11-10T16:14:20,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/c817e390220f42929092dc6d2845694a 2024-11-10T16:14:20,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/df5a495975164396bfe2940c38d3820a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/25165f9beb02406db4901370a419fadd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1c7aeba4a5eb4fa58312aea1fcd50a40, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/55024d2110974d7e8ecab0409e04feeb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/6a382e812ca74c239e176a158b3d1d2d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0bdc4d3b9d7a48638ba7d2e42837d339, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1a73f890ce444907a5c231e41b2bdb6d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c] to archive 2024-11-10T16:14:20,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:20,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/284de5619cad4434a76351e2af149324 2024-11-10T16:14:20,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9f714f48b2f34ea38a1a4fa6c214d306 2024-11-10T16:14:20,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/df5a495975164396bfe2940c38d3820a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/df5a495975164396bfe2940c38d3820a 2024-11-10T16:14:20,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/64a89f0c74f143e7b7320e4aaa5c95a6 2024-11-10T16:14:20,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/8745799a43da49fba0613c3c051b7ccf 2024-11-10T16:14:20,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/d72865e876f841d1bea94737e2b74ea1 2024-11-10T16:14:20,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/25165f9beb02406db4901370a419fadd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/25165f9beb02406db4901370a419fadd 2024-11-10T16:14:20,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0798f75e63b744e0b6257b6a95fce67a 2024-11-10T16:14:20,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/94be0d5e7ff343a0aa90c931611bba92 2024-11-10T16:14:20,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/b7ad207e4080478a894267c528eeb1bf 2024-11-10T16:14:20,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1c7aeba4a5eb4fa58312aea1fcd50a40 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1c7aeba4a5eb4fa58312aea1fcd50a40 2024-11-10T16:14:20,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/cf0077d47ab84530b6603256e037ba8b 2024-11-10T16:14:20,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5b8dc8f89d6742019648009ca6a11d18 2024-11-10T16:14:20,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/55024d2110974d7e8ecab0409e04feeb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/55024d2110974d7e8ecab0409e04feeb 2024-11-10T16:14:20,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4a44760c4ec744c789f36916afd73f32 2024-11-10T16:14:20,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5406b5dbc08b4741805847bc1af49402 2024-11-10T16:14:20,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/6a382e812ca74c239e176a158b3d1d2d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/6a382e812ca74c239e176a158b3d1d2d 2024-11-10T16:14:20,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/01f393e09eac45f29477dd3c098ed9b0 2024-11-10T16:14:20,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/5f149a34e14146bdbef7d059f75c93fb 2024-11-10T16:14:20,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1e52a2ee66b6401ab7905d7bf63d33e1 2024-11-10T16:14:20,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0bdc4d3b9d7a48638ba7d2e42837d339 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/0bdc4d3b9d7a48638ba7d2e42837d339 2024-11-10T16:14:20,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/de0e17737bb94ed69830ec095fadc811 2024-11-10T16:14:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/4323aa27849b41a5b28772ad585c21b6 2024-11-10T16:14:20,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1a73f890ce444907a5c231e41b2bdb6d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1a73f890ce444907a5c231e41b2bdb6d 2024-11-10T16:14:20,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/1f9434be652744a9829a96b79cdc687b 2024-11-10T16:14:20,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9bcd3aec51f54eb7891d2ffc5d7b4ecc 2024-11-10T16:14:20,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/fd776144292845b3b14b5c5ec3836e9c 2024-11-10T16:14:20,374 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits/473.seqid, newMaxSeqId=473, maxSeqId=4 2024-11-10T16:14:20,375 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. 2024-11-10T16:14:20,375 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] regionserver.HRegion(1676): Region close journal for 54d3f36a627b699150cc523f346a55d2: Waiting for close lock at 1731255258617Running coprocessor pre-close hooks at 1731255258617Disabling compacts and flushes for region at 1731255258617Disabling writes for close at 1731255258617Obtaining lock to block concurrent updates at 1731255258617Preparing flush snapshotting stores in 54d3f36a627b699150cc523f346a55d2 at 1731255258617Finished memstore snapshotting TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731255258618 (+1 ms)Flushing stores of TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2. at 1731255258618Flushing 54d3f36a627b699150cc523f346a55d2/A: creating writer at 1731255258618Flushing mob file 54d3f36a627b699150cc523f346a55d2/A: appending metadata at 1731255258624 (+6 ms)Flushing mob file 54d3f36a627b699150cc523f346a55d2/A: closing flushed file at 1731255258624Flushing 54d3f36a627b699150cc523f346a55d2/A: appending metadata at 1731255259039 (+415 ms)Flushing 54d3f36a627b699150cc523f346a55d2/A: closing flushed file at 1731255259039Flushing 54d3f36a627b699150cc523f346a55d2/B: creating writer at 1731255259448 (+409 ms)Flushing 54d3f36a627b699150cc523f346a55d2/B: appending metadata at 1731255259450 (+2 ms)Flushing 54d3f36a627b699150cc523f346a55d2/B: closing flushed file at 1731255259450Flushing 54d3f36a627b699150cc523f346a55d2/C: creating writer at 1731255259858 (+408 ms)Flushing 54d3f36a627b699150cc523f346a55d2/C: appending metadata at 1731255259861 (+3 ms)Flushing 54d3f36a627b699150cc523f346a55d2/C: closing flushed file at 1731255259861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a1944e9: reopening flushed file at 1731255260269 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68072dad: reopening flushed file at 1731255260274 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b907425: reopening flushed file at 1731255260278 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 54d3f36a627b699150cc523f346a55d2 in 1665ms, sequenceid=470, compaction requested=true at 1731255260282 (+4 ms)Writing region close event to WAL at 1731255260371 (+89 ms)Running coprocessor post-close hooks at 1731255260375 (+4 ms)Closed at 1731255260375 2024-11-10T16:14:20,377 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=63}] handler.UnassignRegionHandler(157): Closed 54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=54d3f36a627b699150cc523f346a55d2, regionState=CLOSED 2024-11-10T16:14:20,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:20,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T16:14:20,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseRegionProcedure 54d3f36a627b699150cc523f346a55d2, server=8e3de9160f9f,44105,1731255200149 in 1.9140 sec 2024-11-10T16:14:20,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-10T16:14:20,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54d3f36a627b699150cc523f346a55d2, UNASSIGN in 1.9190 sec 2024-11-10T16:14:20,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-10T16:14:20,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9220 sec 2024-11-10T16:14:20,385 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255260385"}]},"ts":"1731255260385"} 2024-11-10T16:14:20,386 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:14:20,386 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:14:20,388 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-11-10T16:14:20,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T16:14:20,603 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:20,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=64, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T16:14:20,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=64, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,608 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits] 2024-11-10T16:14:20,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/37543ea0025643bcaa1bafee579a3999 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/37543ea0025643bcaa1bafee579a3999 2024-11-10T16:14:20,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/69be8fa10eb047d3b33c5d713cc7a510 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/69be8fa10eb047d3b33c5d713cc7a510 2024-11-10T16:14:20,615 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b96d3be995c341369e1dadfccbcd4edd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/A/b96d3be995c341369e1dadfccbcd4edd 2024-11-10T16:14:20,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/1c4030648e9045d1b045493afdc57c34 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/1c4030648e9045d1b045493afdc57c34 2024-11-10T16:14:20,618 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/573a45452b6a4723b09c5a46c0c108c9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/573a45452b6a4723b09c5a46c0c108c9 2024-11-10T16:14:20,619 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a63fbba2236f4859a95d49fc5e11f0bf to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/B/a63fbba2236f4859a95d49fc5e11f0bf 2024-11-10T16:14:20,621 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9566557a0bd24d649eca6406c288ba97 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/9566557a0bd24d649eca6406c288ba97 2024-11-10T16:14:20,623 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/da175de54ab44bcb9feee22917b61297 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/da175de54ab44bcb9feee22917b61297 2024-11-10T16:14:20,624 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/dd9f92ef9bd6497b975caf78d1ae9040 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/C/dd9f92ef9bd6497b975caf78d1ae9040 2024-11-10T16:14:20,626 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits/473.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2/recovered.edits/473.seqid 2024-11-10T16:14:20,627 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,627 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:14:20,627 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:14:20,628 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T16:14:20,632 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110079f7a86d9d94c1daa790b1ba39c9da4_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110079f7a86d9d94c1daa790b1ba39c9da4_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,633 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015c25b94db3240dfbe723c52d905d227_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111015c25b94db3240dfbe723c52d905d227_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167d147ed6f740f58026e3144d147698_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110167d147ed6f740f58026e3144d147698_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,635 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111025e32c1232404ef797724da317a1e1dd_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111025e32c1232404ef797724da317a1e1dd_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102cd1b473e04e4b918b6a6cc5a2f112ac_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102cd1b473e04e4b918b6a6cc5a2f112ac_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e32c703742048d796012d8eb0fc8bac_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e32c703742048d796012d8eb0fc8bac_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110307bf7fe96554cae9ed22d1ed8e14038_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110307bf7fe96554cae9ed22d1ed8e14038_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110347d65e57bca45adbe95cdafa838c0ea_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110347d65e57bca45adbe95cdafa838c0ea_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047b33bf9cc794a828569923739633e09_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047b33bf9cc794a828569923739633e09_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b44907a6278d48f3aed93df64fb8026b_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b44907a6278d48f3aed93df64fb8026b_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba09a9bc636b4b6f9e41052e35205916_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba09a9bc636b4b6f9e41052e35205916_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cdf282dcba28490ea96afe71cfb2d22e_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cdf282dcba28490ea96afe71cfb2d22e_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e0fdaa60da364158bd1bb06cd7215cfb_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e0fdaa60da364158bd1bb06cd7215cfb_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e42dffda93ce4ad7aa120a830edc6d07_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e42dffda93ce4ad7aa120a830edc6d07_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,647 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e930ddf5306349b6b01b31050db5df18_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e930ddf5306349b6b01b31050db5df18_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,649 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06bcac80cd426fbf08d5646e70724a_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110eb06bcac80cd426fbf08d5646e70724a_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,650 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed87539e2d0d4037a79fc4988a8e89a3_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed87539e2d0d4037a79fc4988a8e89a3_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,651 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f08de5e2772b44059becfe8061353270_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f08de5e2772b44059becfe8061353270_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,652 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f17326153395448093e39fc0c4cbc4b8_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f17326153395448093e39fc0c4cbc4b8_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,653 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f1bd9ec4b9b3450c83a5ed0567f5b241_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f1bd9ec4b9b3450c83a5ed0567f5b241_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,654 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f24883d4a60f40868397733cc0cc5c5f_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f24883d4a60f40868397733cc0cc5c5f_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3de4823e7124637a4517291977a6a0a_54d3f36a627b699150cc523f346a55d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3de4823e7124637a4517291977a6a0a_54d3f36a627b699150cc523f346a55d2 2024-11-10T16:14:20,656 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:14:20,659 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=64, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,661 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:14:20,663 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:14:20,665 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=64, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,665 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:14:20,665 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255260665"}]},"ts":"9223372036854775807"} 2024-11-10T16:14:20,667 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:14:20,667 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 54d3f36a627b699150cc523f346a55d2, NAME => 'TestAcidGuarantees,,1731255227996.54d3f36a627b699150cc523f346a55d2.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:14:20,667 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:14:20,667 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255260667"}]},"ts":"9223372036854775807"} 2024-11-10T16:14:20,669 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:14:20,670 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=64, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-10T16:14:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T16:14:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:14:20,714 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:20,724 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 245) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286154732_22 at /127.0.0.1:38820 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 402), ProcessCount=11 (was 11), AvailableMemoryMB=7132 (was 7207) 2024-11-10T16:14:20,736 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=345, ProcessCount=11, AvailableMemoryMB=7130 2024-11-10T16:14:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:14:20,737 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:14:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:20,739 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:14:20,740 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:20,740 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 65 2024-11-10T16:14:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:20,740 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:14:20,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742077_1253 (size=1061) 2024-11-10T16:14:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:21,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:21,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:14:21,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742078_1254 (size=53) 2024-11-10T16:14:21,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:21,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:14:21,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f8f59a82b84bc6f58d250116e5f29c9f, disabling compactions & flushes 2024-11-10T16:14:21,555 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. after waiting 0 ms 2024-11-10T16:14:21,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,555 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f8f59a82b84bc6f58d250116e5f29c9f: Waiting for close lock at 1731255261555Disabling compacts and flushes for region at 1731255261555Disabling writes for close at 1731255261555Writing region close event to WAL at 1731255261555Closed at 1731255261555 2024-11-10T16:14:21,556 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:14:21,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255261556"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255261556"}]},"ts":"1731255261556"} 2024-11-10T16:14:21,557 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:14:21,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:14:21,558 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255261558"}]},"ts":"1731255261558"} 2024-11-10T16:14:21,559 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:14:21,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, ASSIGN}] 2024-11-10T16:14:21,560 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, ASSIGN 2024-11-10T16:14:21,561 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:14:21,712 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f8f59a82b84bc6f58d250116e5f29c9f, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:21,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=65, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, ASSIGN because future has completed 2024-11-10T16:14:21,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:21,868 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,868 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7752): Opening region: {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:14:21,868 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,868 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:14:21,869 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7794): checking encryption for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,869 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(7797): checking classloading for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,870 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,871 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:21,871 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f59a82b84bc6f58d250116e5f29c9f columnFamilyName A 2024-11-10T16:14:21,871 DEBUG [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:21,872 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(327): Store=f8f59a82b84bc6f58d250116e5f29c9f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:21,872 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,873 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:21,873 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f59a82b84bc6f58d250116e5f29c9f columnFamilyName B 2024-11-10T16:14:21,873 DEBUG [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:21,873 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(327): Store=f8f59a82b84bc6f58d250116e5f29c9f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:21,873 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:21,874 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:21,874 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f59a82b84bc6f58d250116e5f29c9f columnFamilyName C 2024-11-10T16:14:21,874 DEBUG [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:21,875 INFO [StoreOpener-f8f59a82b84bc6f58d250116e5f29c9f-1 {}] regionserver.HStore(327): Store=f8f59a82b84bc6f58d250116e5f29c9f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:21,875 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,875 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1038): replaying wal for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,875 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,876 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,877 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1048): stopping wal replay for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,877 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1060): Cleaning up temporary data for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,877 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:14:21,878 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1093): writing seq id for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,880 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:14:21,880 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1114): Opened f8f59a82b84bc6f58d250116e5f29c9f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66212343, jitterRate=-0.013359203934669495}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:14:21,880 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:21,882 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegion(1006): Region open journal for f8f59a82b84bc6f58d250116e5f29c9f: Running coprocessor pre-open hook at 1731255261869Writing region info on filesystem at 1731255261869Initializing all the Stores at 1731255261869Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255261870 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255261870Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255261870Cleaning up temporary data from old regions at 1731255261877 (+7 ms)Running coprocessor post-open hooks at 1731255261880 (+3 ms)Region opened successfully at 1731255261882 (+2 ms) 2024-11-10T16:14:21,882 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., pid=67, masterSystemTime=1731255261865 2024-11-10T16:14:21,884 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,884 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=67}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:21,884 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f8f59a82b84bc6f58d250116e5f29c9f, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:21,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:21,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T16:14:21,888 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 in 174 msec 2024-11-10T16:14:21,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-10T16:14:21,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, ASSIGN in 328 msec 2024-11-10T16:14:21,891 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:14:21,891 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255261891"}]},"ts":"1731255261891"} 2024-11-10T16:14:21,892 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:14:21,893 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=65, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:14:21,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-10T16:14:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-11-10T16:14:22,884 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:22,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,886 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28be4a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,888 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42f392af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,889 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,891 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:14:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:14:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@371461de 2024-11-10T16:14:22,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:14:22,893 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:14:22,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43dc8da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169cd5b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,897 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424fe29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,898 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d1c0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,900 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be1fce3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6986004a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,903 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a787dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,904 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@245a4430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,905 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,905 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,905 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,906 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b65fb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,907 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4dd7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,908 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd238bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,910 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b29d4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,911 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,911 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,911 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72076d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,912 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d48253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,913 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,913 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,914 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab69000, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,915 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31e53826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,916 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,917 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@582da546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,918 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df568c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3397c776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:22,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:22,919 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6540aff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:22,921 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd93b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:22,922 DEBUG [Thread-1183 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,922 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,922 DEBUG [Thread-1194 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,922 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,923 DEBUG [Thread-1192 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,923 DEBUG [Thread-1190 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,924 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,924 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T16:14:22,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,925 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:22,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,927 DEBUG [Thread-1179 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,927 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,927 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:22,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:22,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,928 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,928 DEBUG [Thread-1181 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:22,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:22,934 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:14:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:22,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:22,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/58cf7e96d51c4307a3ec0934793cd278 is 50, key is test_row_0/A:col10/1731255262936/Put/seqid=0 2024-11-10T16:14:22,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255262957, max timestamp=1731255262970Num uniques 859; 2024-11-10T16:14:22,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:22,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255262957, max timestamp=1731255262970Num uniques 859; 2024-11-10T16:14:22,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:14:22,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255262957, max timestamp=1731255262970Num uniques 859; 2024-11-10T16:14:22,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:14:22,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742079_1255 (size=12001) 2024-11-10T16:14:23,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255323020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255323025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255323026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255323027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255323028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255323133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255323143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255323143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255323143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255323143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:23,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255323343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255323353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255323353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255323353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255323353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/58cf7e96d51c4307a3ec0934793cd278 2024-11-10T16:14:23,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/53594484b0fa4cbabe86e6977e805f30 is 50, key is test_row_0/B:col10/1731255262936/Put/seqid=0 2024-11-10T16:14:23,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742080_1256 (size=12001) 2024-11-10T16:14:23,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:23,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255323653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255323663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255323663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255323663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255323663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:23,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:23,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/53594484b0fa4cbabe86e6977e805f30 2024-11-10T16:14:23,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:23,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:23,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/8d256aa14bfb43f4b4896d6372f91c04 is 50, key is test_row_0/C:col10/1731255262936/Put/seqid=0 2024-11-10T16:14:23,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742081_1257 (size=12001) 2024-11-10T16:14:24,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:24,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:24,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:24,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255324163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255324174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255324174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255324174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255324174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:24,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/8d256aa14bfb43f4b4896d6372f91c04 2024-11-10T16:14:24,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/58cf7e96d51c4307a3ec0934793cd278 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278 2024-11-10T16:14:24,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T16:14:24,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/53594484b0fa4cbabe86e6977e805f30 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30 2024-11-10T16:14:24,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T16:14:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/8d256aa14bfb43f4b4896d6372f91c04 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04 2024-11-10T16:14:24,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04, entries=150, sequenceid=12, filesize=11.7 K 2024-11-10T16:14:24,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for f8f59a82b84bc6f58d250116e5f29c9f in 1348ms, sequenceid=12, compaction requested=false 2024-11-10T16:14:24,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:24,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T16:14:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,307 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:24,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/14c7357b4c024af899f5fec3cccd593d is 50, key is test_row_0/A:col10/1731255262984/Put/seqid=0 2024-11-10T16:14:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742082_1258 (size=12001) 2024-11-10T16:14:24,719 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/14c7357b4c024af899f5fec3cccd593d 2024-11-10T16:14:24,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0329ee64ee0f4478b784a9c5107249c1 is 50, key is test_row_0/B:col10/1731255262984/Put/seqid=0 2024-11-10T16:14:24,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742083_1259 (size=12001) 2024-11-10T16:14:24,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0329ee64ee0f4478b784a9c5107249c1 2024-11-10T16:14:24,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b0385a0fd6c546308cc3c9f83896be10 is 50, key is test_row_0/C:col10/1731255262984/Put/seqid=0 2024-11-10T16:14:24,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742084_1260 (size=12001) 2024-11-10T16:14:24,767 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b0385a0fd6c546308cc3c9f83896be10 2024-11-10T16:14:24,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/14c7357b4c024af899f5fec3cccd593d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d 2024-11-10T16:14:24,780 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T16:14:24,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0329ee64ee0f4478b784a9c5107249c1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1 2024-11-10T16:14:24,785 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T16:14:24,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b0385a0fd6c546308cc3c9f83896be10 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10 2024-11-10T16:14:24,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T16:14:24,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=0 B/0 for f8f59a82b84bc6f58d250116e5f29c9f in 484ms, sequenceid=49, compaction requested=false 2024-11-10T16:14:24,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:24,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T16:14:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T16:14:24,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T16:14:24,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-10T16:14:24,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.8750 sec 2024-11-10T16:14:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T16:14:25,074 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:25,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:25,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T16:14:25,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:25,077 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:25,078 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:25,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:25,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:25,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:14:25,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:25,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/6c2f42a6c12c416e829f61d0524c1ec7 is 50, key is test_row_0/A:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:25,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742085_1261 (size=12001) 2024-11-10T16:14:25,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255265209, max timestamp=1731255265218Num uniques 859; 2024-11-10T16:14:25,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:25,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255265209, max timestamp=1731255265218Num uniques 859; 2024-11-10T16:14:25,222 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:25,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255265209, max timestamp=1731255265218Num uniques 859; 2024-11-10T16:14:25,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:14:25,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255325265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255325266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255325273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255325273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255325275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255325384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255325384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255325384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:25,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255325394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255325394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255325593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255325593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255325593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255325603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255325604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/6c2f42a6c12c416e829f61d0524c1ec7 2024-11-10T16:14:25,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6ba578df292b4fe5baf09f0b39cf375c is 50, key is test_row_0/B:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:25,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742086_1262 (size=12001) 2024-11-10T16:14:25,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:25,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,903 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:14:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255325903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255325903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255325903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255325913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255325914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:25,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:25,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:25,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6ba578df292b4fe5baf09f0b39cf375c 2024-11-10T16:14:26,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/84039d2a9cda4e8da1c554d72ba11640 is 50, key is test_row_0/C:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:26,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742087_1263 (size=12001) 2024-11-10T16:14:26,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:26,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:26,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:26,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255326413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255326414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255326414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:26,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255326423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255326423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:26,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/84039d2a9cda4e8da1c554d72ba11640 2024-11-10T16:14:26,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/6c2f42a6c12c416e829f61d0524c1ec7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7 2024-11-10T16:14:26,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T16:14:26,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6ba578df292b4fe5baf09f0b39cf375c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c 2024-11-10T16:14:26,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T16:14:26,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/84039d2a9cda4e8da1c554d72ba11640 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640 2024-11-10T16:14:26,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T16:14:26,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for f8f59a82b84bc6f58d250116e5f29c9f in 1263ms, sequenceid=60, compaction requested=true 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:26,456 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:26,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:14:26,457 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:26,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T16:14:26,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,458 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:26,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:26,459 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:26,459 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:26,459 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,459 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.2 K 2024-11-10T16:14:26,459 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:26,459 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:26,459 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,460 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.2 K 2024-11-10T16:14:26,460 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 53594484b0fa4cbabe86e6977e805f30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731255262935 2024-11-10T16:14:26,460 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58cf7e96d51c4307a3ec0934793cd278, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731255262935 2024-11-10T16:14:26,460 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0329ee64ee0f4478b784a9c5107249c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731255262984 2024-11-10T16:14:26,461 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14c7357b4c024af899f5fec3cccd593d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731255262984 2024-11-10T16:14:26,461 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba578df292b4fe5baf09f0b39cf375c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:26,461 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c2f42a6c12c416e829f61d0524c1ec7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/05688cbab9d348a8abf329cebab911be is 50, key is test_row_0/A:col10/1731255265271/Put/seqid=0 2024-11-10T16:14:26,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742088_1264 (size=12001) 2024-11-10T16:14:26,470 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/05688cbab9d348a8abf329cebab911be 2024-11-10T16:14:26,472 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:26,473 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:26,473 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/1ac4d31cf7014e378314171eb85dba30 is 50, key is test_row_0/B:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:26,473 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/84ede8172ce146c0a419b6091876228e is 50, key is test_row_0/A:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:26,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742089_1265 (size=12104) 2024-11-10T16:14:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/914e0b1e6d0b44918a2a7e97a6b22287 is 50, key is test_row_0/B:col10/1731255265271/Put/seqid=0 2024-11-10T16:14:26,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742090_1266 (size=12104) 2024-11-10T16:14:26,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742091_1267 (size=12001) 2024-11-10T16:14:26,884 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/1ac4d31cf7014e378314171eb85dba30 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1ac4d31cf7014e378314171eb85dba30 2024-11-10T16:14:26,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/914e0b1e6d0b44918a2a7e97a6b22287 2024-11-10T16:14:26,889 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/84ede8172ce146c0a419b6091876228e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/84ede8172ce146c0a419b6091876228e 2024-11-10T16:14:26,890 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 1ac4d31cf7014e378314171eb85dba30(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:26,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:26,890 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255266456; duration=0sec 2024-11-10T16:14:26,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:26,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:26,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:26,892 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:26,892 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:26,892 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:26,892 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.2 K 2024-11-10T16:14:26,893 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d256aa14bfb43f4b4896d6372f91c04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731255262935 2024-11-10T16:14:26,895 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b0385a0fd6c546308cc3c9f83896be10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731255262984 2024-11-10T16:14:26,895 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 84ede8172ce146c0a419b6091876228e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:26,895 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:26,895 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255266456; duration=0sec 2024-11-10T16:14:26,895 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:26,895 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:26,896 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 84039d2a9cda4e8da1c554d72ba11640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:26,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/fcad781cf9804af892c5a420dd8ec47d is 50, key is test_row_0/C:col10/1731255265271/Put/seqid=0 2024-11-10T16:14:26,905 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:26,906 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/fb67428753f442cbb2a4535046572b6c is 50, key is test_row_0/C:col10/1731255265193/Put/seqid=0 2024-11-10T16:14:26,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742092_1268 (size=12001) 2024-11-10T16:14:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742093_1269 (size=12104) 2024-11-10T16:14:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:27,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/fcad781cf9804af892c5a420dd8ec47d 2024-11-10T16:14:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/05688cbab9d348a8abf329cebab911be as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be 2024-11-10T16:14:27,315 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/fb67428753f442cbb2a4535046572b6c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fb67428753f442cbb2a4535046572b6c 2024-11-10T16:14:27,319 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T16:14:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/914e0b1e6d0b44918a2a7e97a6b22287 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287 2024-11-10T16:14:27,321 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into fb67428753f442cbb2a4535046572b6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:27,321 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:27,322 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255266456; duration=0sec 2024-11-10T16:14:27,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:27,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:27,325 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T16:14:27,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/fcad781cf9804af892c5a420dd8ec47d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d 2024-11-10T16:14:27,332 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d, entries=150, sequenceid=95, filesize=11.7 K 2024-11-10T16:14:27,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=0 B/0 for f8f59a82b84bc6f58d250116e5f29c9f in 875ms, sequenceid=95, compaction requested=false 2024-11-10T16:14:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T16:14:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T16:14:27,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T16:14:27,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2560 sec 2024-11-10T16:14:27,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2630 sec 2024-11-10T16:14:27,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:27,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:27,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:27,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e49cc42e600a4fc0a7b6e959bab0b872 is 50, key is test_row_0/A:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:27,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742094_1270 (size=12001) 2024-11-10T16:14:27,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255327458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255267455, max timestamp=1731255267462Num uniques 859; 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255267455, max timestamp=1731255267462Num uniques 859; 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255267455, max timestamp=1731255267462Num uniques 859; 2024-11-10T16:14:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:14:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255327483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255327487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255327488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255327489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255327573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255327593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255327603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255327603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255327604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255327783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255327803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255327814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255327814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255327814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:27,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e49cc42e600a4fc0a7b6e959bab0b872 2024-11-10T16:14:27,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/1fe4ff1793714f159350b1c47d791fda is 50, key is test_row_0/B:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:27,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742095_1271 (size=12001) 2024-11-10T16:14:28,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255328093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255328113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255328123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255328123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255328123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/1fe4ff1793714f159350b1c47d791fda 2024-11-10T16:14:28,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/7f74f35269f44dbdaaec34fc2462ebd2 is 50, key is test_row_0/C:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742096_1272 (size=12001) 2024-11-10T16:14:28,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255328603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255328623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255328633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255328633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255328633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:28,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/7f74f35269f44dbdaaec34fc2462ebd2 2024-11-10T16:14:28,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e49cc42e600a4fc0a7b6e959bab0b872 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872 2024-11-10T16:14:28,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T16:14:28,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/1fe4ff1793714f159350b1c47d791fda as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda 2024-11-10T16:14:28,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T16:14:28,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/7f74f35269f44dbdaaec34fc2462ebd2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2 2024-11-10T16:14:28,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2, entries=150, sequenceid=109, filesize=11.7 K 2024-11-10T16:14:28,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f8f59a82b84bc6f58d250116e5f29c9f in 1266ms, sequenceid=109, compaction requested=true 2024-11-10T16:14:28,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:28,706 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:28,706 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:28,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:28,707 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:28,707 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:28,707 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/84ede8172ce146c0a419b6091876228e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.3 K 2024-11-10T16:14:28,707 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1ac4d31cf7014e378314171eb85dba30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.3 K 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ac4d31cf7014e378314171eb85dba30, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:28,707 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ede8172ce146c0a419b6091876228e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:28,708 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 914e0b1e6d0b44918a2a7e97a6b22287, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731255265271 2024-11-10T16:14:28,708 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05688cbab9d348a8abf329cebab911be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731255265271 2024-11-10T16:14:28,708 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e49cc42e600a4fc0a7b6e959bab0b872, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:28,708 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fe4ff1793714f159350b1c47d791fda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:28,717 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:28,717 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:28,717 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/19d8acea038e411a9668e135d9fd51f8 is 50, key is test_row_0/A:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:28,717 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/32cfbb066a464c3abec1b01aec27102f is 50, key is test_row_0/B:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:28,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742097_1273 (size=12207) 2024-11-10T16:14:28,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742098_1274 (size=12207) 2024-11-10T16:14:29,127 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/32cfbb066a464c3abec1b01aec27102f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/32cfbb066a464c3abec1b01aec27102f 2024-11-10T16:14:29,133 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 32cfbb066a464c3abec1b01aec27102f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:29,133 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:29,133 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255268706; duration=0sec 2024-11-10T16:14:29,133 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:29,133 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:29,133 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:29,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:29,135 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:29,135 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/19d8acea038e411a9668e135d9fd51f8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/19d8acea038e411a9668e135d9fd51f8 2024-11-10T16:14:29,136 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:29,136 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fb67428753f442cbb2a4535046572b6c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.3 K 2024-11-10T16:14:29,137 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fb67428753f442cbb2a4535046572b6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731255265183 2024-11-10T16:14:29,138 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fcad781cf9804af892c5a420dd8ec47d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731255265271 2024-11-10T16:14:29,138 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f74f35269f44dbdaaec34fc2462ebd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:29,141 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 19d8acea038e411a9668e135d9fd51f8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:29,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:29,141 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255268705; duration=0sec 2024-11-10T16:14:29,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:29,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:29,151 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:29,151 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/bddc3199ee4347a78bdb27e856251141 is 50, key is test_row_0/C:col10/1731255267439/Put/seqid=0 2024-11-10T16:14:29,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742099_1275 (size=12207) 2024-11-10T16:14:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T16:14:29,244 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:29,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T16:14:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:29,247 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:29,247 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:29,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:29,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T16:14:29,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:29,402 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:29,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:29,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b7d8627da21f447b882ca64097106e9f is 50, key is test_row_0/A:col10/1731255267488/Put/seqid=0 2024-11-10T16:14:29,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742100_1276 (size=12151) 2024-11-10T16:14:29,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:29,564 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/bddc3199ee4347a78bdb27e856251141 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/bddc3199ee4347a78bdb27e856251141 2024-11-10T16:14:29,569 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into bddc3199ee4347a78bdb27e856251141(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:29,569 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:29,569 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255268706; duration=0sec 2024-11-10T16:14:29,569 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:29,569 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:29,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:29,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255329643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,671 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255329653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255329671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,677 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:29,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255329671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255329671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255329783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255329783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255329783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,815 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b7d8627da21f447b882ca64097106e9f 2024-11-10T16:14:29,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/80491afe940149d1a5d7dd3498d9c8a9 is 50, key is test_row_0/B:col10/1731255267488/Put/seqid=0 2024-11-10T16:14:29,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742101_1277 (size=12151) 2024-11-10T16:14:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:29,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255329993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255329993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:29,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255329994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:29,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,228 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/80491afe940149d1a5d7dd3498d9c8a9 2024-11-10T16:14:30,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c0f9023ab37458d9f418b349ae64893 is 50, key is test_row_0/C:col10/1731255267488/Put/seqid=0 2024-11-10T16:14:30,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742102_1278 (size=12151) 2024-11-10T16:14:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255330303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255330303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255330303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:30,641 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c0f9023ab37458d9f418b349ae64893 2024-11-10T16:14:30,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b7d8627da21f447b882ca64097106e9f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f 2024-11-10T16:14:30,650 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T16:14:30,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/80491afe940149d1a5d7dd3498d9c8a9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9 2024-11-10T16:14:30,655 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T16:14:30,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c0f9023ab37458d9f418b349ae64893 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893 2024-11-10T16:14:30,660 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893, entries=150, sequenceid=146, filesize=11.9 K 2024-11-10T16:14:30,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for f8f59a82b84bc6f58d250116e5f29c9f in 1260ms, sequenceid=146, compaction requested=false 2024-11-10T16:14:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:30,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T16:14:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T16:14:30,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T16:14:30,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-10T16:14:30,667 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4210 sec 2024-11-10T16:14:30,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:30,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:30,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:30,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b013ac4eb13649dc8e162ae311ca5978 is 50, key is test_row_0/A:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:30,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742103_1279 (size=16931) 2024-11-10T16:14:30,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255330869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255330874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255330875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255330983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255330983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255330983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:30,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255331193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255331193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255331193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b013ac4eb13649dc8e162ae311ca5978 2024-11-10T16:14:31,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/86375dc4fa8142c49d6e0d4f399c1249 is 50, key is test_row_0/B:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742104_1280 (size=12151) 2024-11-10T16:14:31,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T16:14:31,403 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:31,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T16:14:31,406 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:31,406 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:31,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:31,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255331503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255331503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255331503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:31,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T16:14:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/86375dc4fa8142c49d6e0d4f399c1249 2024-11-10T16:14:31,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dd26a8a84d654ccab3c6da5f25cc8cc2 is 50, key is test_row_0/C:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:31,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742105_1281 (size=12151) 2024-11-10T16:14:31,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255331693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,697 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255331703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:31,707 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:31,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:31,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T16:14:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:31,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T16:14:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:31,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:31,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:32,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255332013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255332013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:32,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:32,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255332023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:32,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T16:14:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:32,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:32,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:32,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dd26a8a84d654ccab3c6da5f25cc8cc2 2024-11-10T16:14:32,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/b013ac4eb13649dc8e162ae311ca5978 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978 2024-11-10T16:14:32,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978, entries=250, sequenceid=161, filesize=16.5 K 2024-11-10T16:14:32,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/86375dc4fa8142c49d6e0d4f399c1249 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249 2024-11-10T16:14:32,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T16:14:32,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dd26a8a84d654ccab3c6da5f25cc8cc2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2 2024-11-10T16:14:32,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2, entries=150, sequenceid=161, filesize=11.9 K 2024-11-10T16:14:32,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f8f59a82b84bc6f58d250116e5f29c9f in 1259ms, sequenceid=161, compaction requested=true 2024-11-10T16:14:32,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:32,077 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:32,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:14:32,077 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:32,078 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:32,078 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:32,078 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,078 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/19d8acea038e411a9668e135d9fd51f8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=40.3 K 2024-11-10T16:14:32,078 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:32,078 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19d8acea038e411a9668e135d9fd51f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:32,078 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:32,078 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,078 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/32cfbb066a464c3abec1b01aec27102f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.7 K 2024-11-10T16:14:32,079 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d8627da21f447b882ca64097106e9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731255267485 2024-11-10T16:14:32,079 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cfbb066a464c3abec1b01aec27102f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:32,079 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b013ac4eb13649dc8e162ae311ca5978, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:32,079 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 80491afe940149d1a5d7dd3498d9c8a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731255267485 2024-11-10T16:14:32,080 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 86375dc4fa8142c49d6e0d4f399c1249, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:32,088 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:32,090 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/4e59546c98c34ce2a01c5fa0720cad10 is 50, key is test_row_0/B:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:32,091 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:32,091 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/4d643a85105a48429b3b36489e58606e is 50, key is test_row_0/A:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:32,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742106_1282 (size=12459) 2024-11-10T16:14:32,102 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/4e59546c98c34ce2a01c5fa0720cad10 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4e59546c98c34ce2a01c5fa0720cad10 2024-11-10T16:14:32,107 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 4e59546c98c34ce2a01c5fa0720cad10(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:32,107 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:32,107 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255272077; duration=0sec 2024-11-10T16:14:32,108 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:32,108 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:32,109 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:32,110 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:32,110 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:32,110 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,110 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/bddc3199ee4347a78bdb27e856251141, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.7 K 2024-11-10T16:14:32,111 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting bddc3199ee4347a78bdb27e856251141, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731255267437 2024-11-10T16:14:32,111 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c0f9023ab37458d9f418b349ae64893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731255267485 2024-11-10T16:14:32,111 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting dd26a8a84d654ccab3c6da5f25cc8cc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742107_1283 (size=12459) 2024-11-10T16:14:32,119 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/4d643a85105a48429b3b36489e58606e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/4d643a85105a48429b3b36489e58606e 2024-11-10T16:14:32,123 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:32,124 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/64987fb634044486aaf1c505dfb5392f is 50, key is test_row_0/C:col10/1731255269641/Put/seqid=0 2024-11-10T16:14:32,126 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 4d643a85105a48429b3b36489e58606e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:32,126 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:32,127 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255272077; duration=0sec 2024-11-10T16:14:32,127 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:32,127 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742108_1284 (size=12459) 2024-11-10T16:14:32,153 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/64987fb634044486aaf1c505dfb5392f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/64987fb634044486aaf1c505dfb5392f 2024-11-10T16:14:32,161 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into 64987fb634044486aaf1c505dfb5392f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:32,161 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:32,162 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255272077; duration=0sec 2024-11-10T16:14:32,162 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:32,162 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:32,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T16:14:32,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:32,180 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:32,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7a0f0884f2f946a798741f9ffaee2ee1 is 50, key is test_row_0/A:col10/1731255270873/Put/seqid=0 2024-11-10T16:14:32,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742109_1285 (size=12151) 2024-11-10T16:14:32,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7a0f0884f2f946a798741f9ffaee2ee1 2024-11-10T16:14:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/683470be7614413da511a5d70a3741f2 is 50, key is test_row_0/B:col10/1731255270873/Put/seqid=0 2024-11-10T16:14:32,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742110_1286 (size=12151) 2024-11-10T16:14:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:32,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/683470be7614413da511a5d70a3741f2 2024-11-10T16:14:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/adc69eb164f440a895aef2aee7251f5b is 50, key is test_row_0/C:col10/1731255270873/Put/seqid=0 2024-11-10T16:14:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742111_1287 (size=12151) 2024-11-10T16:14:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:33,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:33,040 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/adc69eb164f440a895aef2aee7251f5b 2024-11-10T16:14:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7a0f0884f2f946a798741f9ffaee2ee1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1 2024-11-10T16:14:33,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T16:14:33,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/683470be7614413da511a5d70a3741f2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2 2024-11-10T16:14:33,055 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T16:14:33,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255333050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255333051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/adc69eb164f440a895aef2aee7251f5b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b 2024-11-10T16:14:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255333055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T16:14:33,064 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f8f59a82b84bc6f58d250116e5f29c9f in 885ms, sequenceid=182, compaction requested=false 2024-11-10T16:14:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:33,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T16:14:33,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T16:14:33,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T16:14:33,067 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-11-10T16:14:33,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.6650 sec 2024-11-10T16:14:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:33,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:33,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:33,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/0adbe053e32a42fa8f2b0b9cf5529f30 is 50, key is test_row_0/A:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:33,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742112_1288 (size=12151) 2024-11-10T16:14:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255333186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255333190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255333190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255333293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255333303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255333304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255333503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255333513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255333513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T16:14:33,553 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:33,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T16:14:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:33,555 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:33,556 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:33,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:33,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/0adbe053e32a42fa8f2b0b9cf5529f30 2024-11-10T16:14:33,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5b86a143cfb14dbb826c1c6c291925b5 is 50, key is test_row_0/B:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:33,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742113_1289 (size=12151) 2024-11-10T16:14:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:33,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:33,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255333813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255333823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255333823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:33,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:33,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:33,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:33,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5b86a143cfb14dbb826c1c6c291925b5 2024-11-10T16:14:33,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b81f268431bb4d748bbaa3d6c53ced16 is 50, key is test_row_0/C:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:33,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742114_1290 (size=12151) 2024-11-10T16:14:34,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:34,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:34,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:34,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255334323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255334333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255334333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:34,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b81f268431bb4d748bbaa3d6c53ced16 2024-11-10T16:14:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/0adbe053e32a42fa8f2b0b9cf5529f30 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30 2024-11-10T16:14:34,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30, entries=150, sequenceid=197, filesize=11.9 K 2024-11-10T16:14:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5b86a143cfb14dbb826c1c6c291925b5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5 2024-11-10T16:14:34,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5, entries=150, sequenceid=197, filesize=11.9 K 2024-11-10T16:14:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b81f268431bb4d748bbaa3d6c53ced16 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16 2024-11-10T16:14:34,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16, entries=150, sequenceid=197, filesize=11.9 K 2024-11-10T16:14:34,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f8f59a82b84bc6f58d250116e5f29c9f in 1254ms, sequenceid=197, compaction requested=true 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:34,420 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:34,420 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:34,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:34,421 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:34,421 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:34,421 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:34,421 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:34,421 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,421 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,421 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4e59546c98c34ce2a01c5fa0720cad10, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.9 K 2024-11-10T16:14:34,421 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/4d643a85105a48429b3b36489e58606e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.9 K 2024-11-10T16:14:34,422 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e59546c98c34ce2a01c5fa0720cad10, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:34,422 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d643a85105a48429b3b36489e58606e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:34,422 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 683470be7614413da511a5d70a3741f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255270868 2024-11-10T16:14:34,422 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0f0884f2f946a798741f9ffaee2ee1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255270868 2024-11-10T16:14:34,423 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b86a143cfb14dbb826c1c6c291925b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:34,423 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0adbe053e32a42fa8f2b0b9cf5529f30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:34,430 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:34,430 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e48457e5351b4cb684c3592461896a58 is 50, key is test_row_0/A:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:34,431 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:34,431 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/9f621dbee4a54ba08aac398b7a6d7723 is 50, key is test_row_0/B:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:34,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742116_1292 (size=12561) 2024-11-10T16:14:34,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742115_1291 (size=12561) 2024-11-10T16:14:34,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T16:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 is 50, key is test_row_0/A:col10/1731255273189/Put/seqid=0 2024-11-10T16:14:34,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742117_1293 (size=12151) 2024-11-10T16:14:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:34,872 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/9f621dbee4a54ba08aac398b7a6d7723 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/9f621dbee4a54ba08aac398b7a6d7723 2024-11-10T16:14:34,876 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e48457e5351b4cb684c3592461896a58 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e48457e5351b4cb684c3592461896a58 2024-11-10T16:14:34,879 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 9f621dbee4a54ba08aac398b7a6d7723(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:34,879 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:34,879 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255274420; duration=0sec 2024-11-10T16:14:34,879 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:34,879 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:34,879 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:34,880 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:34,881 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:34,881 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:34,881 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/64987fb634044486aaf1c505dfb5392f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=35.9 K 2024-11-10T16:14:34,881 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 64987fb634044486aaf1c505dfb5392f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1731255269641 2024-11-10T16:14:34,882 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting adc69eb164f440a895aef2aee7251f5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255270868 2024-11-10T16:14:34,882 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into e48457e5351b4cb684c3592461896a58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:34,882 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:34,882 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255274420; duration=0sec 2024-11-10T16:14:34,882 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:34,882 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:34,882 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b81f268431bb4d748bbaa3d6c53ced16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:34,889 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:34,889 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 2024-11-10T16:14:34,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/79abd6fe1bfd48b3a09a590084422520 is 50, key is test_row_0/C:col10/1731255273050/Put/seqid=0 2024-11-10T16:14:34,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6b82dfa0723b490f83ad934c14897fc8 is 50, key is test_row_0/B:col10/1731255273189/Put/seqid=0 2024-11-10T16:14:34,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742118_1294 (size=12561) 2024-11-10T16:14:34,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742119_1295 (size=12151) 2024-11-10T16:14:35,304 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6b82dfa0723b490f83ad934c14897fc8 2024-11-10T16:14:35,307 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/79abd6fe1bfd48b3a09a590084422520 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/79abd6fe1bfd48b3a09a590084422520 2024-11-10T16:14:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/45aa596da4bf4fda96819e84d9be1348 is 50, key is test_row_0/C:col10/1731255273189/Put/seqid=0 2024-11-10T16:14:35,312 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into 79abd6fe1bfd48b3a09a590084422520(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:35,313 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:35,313 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255274420; duration=0sec 2024-11-10T16:14:35,313 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:35,313 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:35,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742120_1296 (size=12151) 2024-11-10T16:14:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:35,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:35,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255335358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255335362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255335361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255335473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255335474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255335474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255335683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255335683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255335683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:35,717 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/45aa596da4bf4fda96819e84d9be1348 2024-11-10T16:14:35,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 2024-11-10T16:14:35,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49762 deadline: 1731255335723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:35,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T16:14:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49794 deadline: 1731255335723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:35,726 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/6b82dfa0723b490f83ad934c14897fc8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8 2024-11-10T16:14:35,726 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in f8f59a82b84bc6f58d250116e5f29c9f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:35,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:35,731 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T16:14:35,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/45aa596da4bf4fda96819e84d9be1348 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348 2024-11-10T16:14:35,735 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348, entries=150, sequenceid=214, filesize=11.9 K 2024-11-10T16:14:35,736 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f8f59a82b84bc6f58d250116e5f29c9f in 1262ms, sequenceid=214, compaction requested=false 2024-11-10T16:14:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:35,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T16:14:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T16:14:35,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T16:14:35,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-10T16:14:35,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1860 sec 2024-11-10T16:14:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:35,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:35,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:36,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/db0a706dbcf94498a12641dfea7bf4ab is 50, key is test_row_0/A:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:36,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742121_1297 (size=12151) 2024-11-10T16:14:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255336017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255336018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255336023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255336134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255336134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255336134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255336343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255336344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255336344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/db0a706dbcf94498a12641dfea7bf4ab 2024-11-10T16:14:36,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/69f37a62106b4cbcb25e406f84eda5c5 is 50, key is test_row_0/B:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:36,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742122_1298 (size=12151) 2024-11-10T16:14:36,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255336653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255336653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255336653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:36,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:36,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/69f37a62106b4cbcb25e406f84eda5c5 2024-11-10T16:14:36,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b5b174d08f7740f59c3baae79064d415 is 50, key is test_row_0/C:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:36,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742123_1299 (size=12151) 2024-11-10T16:14:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255337163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255337163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:37,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:37,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255337163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:37,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b5b174d08f7740f59c3baae79064d415 2024-11-10T16:14:37,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/db0a706dbcf94498a12641dfea7bf4ab as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab 2024-11-10T16:14:37,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T16:14:37,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/69f37a62106b4cbcb25e406f84eda5c5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5 2024-11-10T16:14:37,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T16:14:37,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b5b174d08f7740f59c3baae79064d415 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415 2024-11-10T16:14:37,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T16:14:37,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f8f59a82b84bc6f58d250116e5f29c9f in 1257ms, sequenceid=233, compaction requested=true 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:37,253 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:37,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:37,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:37,254 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:37,254 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:37,254 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:37,254 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:37,254 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:37,254 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:37,255 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:37,255 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e48457e5351b4cb684c3592461896a58, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.0 K 2024-11-10T16:14:37,255 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/9f621dbee4a54ba08aac398b7a6d7723, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.0 K 2024-11-10T16:14:37,255 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e48457e5351b4cb684c3592461896a58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:37,255 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f621dbee4a54ba08aac398b7a6d7723, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:37,255 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b82dfa0723b490f83ad934c14897fc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255273184 2024-11-10T16:14:37,255 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ecfe1e1de0a4f32885cbc8cfcfd0d98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255273184 2024-11-10T16:14:37,256 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f37a62106b4cbcb25e406f84eda5c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:37,256 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting db0a706dbcf94498a12641dfea7bf4ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:37,267 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:37,267 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:37,267 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/159dca58d917433591552859768845f4 is 50, key is test_row_0/B:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:37,267 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/3fe82f1ff09f4085a1b7d6b43693f000 is 50, key is test_row_0/A:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:37,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742124_1300 (size=12663) 2024-11-10T16:14:37,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742125_1301 (size=12663) 2024-11-10T16:14:37,678 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/3fe82f1ff09f4085a1b7d6b43693f000 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/3fe82f1ff09f4085a1b7d6b43693f000 2024-11-10T16:14:37,678 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/159dca58d917433591552859768845f4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/159dca58d917433591552859768845f4 2024-11-10T16:14:37,683 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 3fe82f1ff09f4085a1b7d6b43693f000(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:37,683 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:37,683 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255277253; duration=0sec 2024-11-10T16:14:37,683 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:37,683 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:37,684 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:37,684 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 159dca58d917433591552859768845f4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:37,684 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:37,684 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255277253; duration=0sec 2024-11-10T16:14:37,684 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:37,684 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:37,685 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:37,685 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:37,685 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:37,685 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/79abd6fe1bfd48b3a09a590084422520, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.0 K 2024-11-10T16:14:37,685 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79abd6fe1bfd48b3a09a590084422520, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731255273046 2024-11-10T16:14:37,686 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45aa596da4bf4fda96819e84d9be1348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731255273184 2024-11-10T16:14:37,686 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5b174d08f7740f59c3baae79064d415, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:37,694 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:37,695 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/a27c34650a134244972ed5f0d752f570 is 50, key is test_row_0/C:col10/1731255275360/Put/seqid=0 2024-11-10T16:14:37,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742126_1302 (size=12663) 2024-11-10T16:14:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T16:14:37,713 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:37,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T16:14:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:37,715 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:37,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:37,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:37,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:37,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:37,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:37,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/91fa5054220649a3815f536e03d9e2dd is 50, key is test_row_0/A:col10/1731255276017/Put/seqid=0 2024-11-10T16:14:37,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742127_1303 (size=12151) 2024-11-10T16:14:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:38,103 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/a27c34650a134244972ed5f0d752f570 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/a27c34650a134244972ed5f0d752f570 2024-11-10T16:14:38,108 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into a27c34650a134244972ed5f0d752f570(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:38,108 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:38,108 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255277253; duration=0sec 2024-11-10T16:14:38,108 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:38,108 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:38,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255338192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255338193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255338195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,280 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/91fa5054220649a3815f536e03d9e2dd 2024-11-10T16:14:38,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/33cd705eb5894299b061dc31ead4b2e6 is 50, key is test_row_0/B:col10/1731255276017/Put/seqid=0 2024-11-10T16:14:38,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742128_1304 (size=12151) 2024-11-10T16:14:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255338303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255338303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255338303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:38,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255338513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255338513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255338513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,694 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/33cd705eb5894299b061dc31ead4b2e6 2024-11-10T16:14:38,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e22b585fbac545649c9ff9b3fa969947 is 50, key is test_row_0/C:col10/1731255276017/Put/seqid=0 2024-11-10T16:14:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742129_1305 (size=12151) 2024-11-10T16:14:38,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255338823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255338823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255338823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:38,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:39,106 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e22b585fbac545649c9ff9b3fa969947 2024-11-10T16:14:39,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/91fa5054220649a3815f536e03d9e2dd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd 2024-11-10T16:14:39,115 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd, entries=150, sequenceid=251, filesize=11.9 K 2024-11-10T16:14:39,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/33cd705eb5894299b061dc31ead4b2e6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6 2024-11-10T16:14:39,121 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6, entries=150, sequenceid=251, filesize=11.9 K 2024-11-10T16:14:39,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e22b585fbac545649c9ff9b3fa969947 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947 2024-11-10T16:14:39,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947, entries=150, sequenceid=251, filesize=11.9 K 2024-11-10T16:14:39,127 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for f8f59a82b84bc6f58d250116e5f29c9f in 1257ms, sequenceid=251, compaction requested=false 2024-11-10T16:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T16:14:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T16:14:39,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T16:14:39,131 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-10T16:14:39,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4180 sec 2024-11-10T16:14:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:39,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:39,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:39,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/336986749d804c3886511687f3bc1c5a is 50, key is test_row_0/A:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742130_1306 (size=12301) 2024-11-10T16:14:39,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255339357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255339357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255339358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255339473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255339473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255339473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255339684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255339684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255339684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/336986749d804c3886511687f3bc1c5a 2024-11-10T16:14:39,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/200ad0be388b452c8471f2f26c06abbd is 50, key is test_row_0/B:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:39,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742131_1307 (size=12301) 2024-11-10T16:14:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T16:14:39,863 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:39,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T16:14:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:39,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:39,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:39,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255339993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255339993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255339993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:39,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T16:14:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/200ad0be388b452c8471f2f26c06abbd 2024-11-10T16:14:40,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/c6e4d3a12cb44c2a9ab4a629775f1807 is 50, key is test_row_0/C:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:40,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T16:14:40,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:40,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:40,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742132_1308 (size=12301) 2024-11-10T16:14:40,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:40,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/c6e4d3a12cb44c2a9ab4a629775f1807 2024-11-10T16:14:40,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/336986749d804c3886511687f3bc1c5a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a 2024-11-10T16:14:40,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-10T16:14:40,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/200ad0be388b452c8471f2f26c06abbd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd 2024-11-10T16:14:40,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd, entries=150, sequenceid=270, filesize=12.0 K 2024-11-10T16:14:40,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/c6e4d3a12cb44c2a9ab4a629775f1807 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807 2024-11-10T16:14:40,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807, entries=150, sequenceid=270, filesize=12.0 K 2024-11-10T16:14:40,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for f8f59a82b84bc6f58d250116e5f29c9f in 875ms, sequenceid=270, compaction requested=true 2024-11-10T16:14:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:40,211 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:40,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:40,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-10T16:14:40,212 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:40,212 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:40,212 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:40,212 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,213 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/3fe82f1ff09f4085a1b7d6b43693f000, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.2 K 2024-11-10T16:14:40,214 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe82f1ff09f4085a1b7d6b43693f000, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:40,214 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:40,214 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:40,214 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,214 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/159dca58d917433591552859768845f4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.2 K 2024-11-10T16:14:40,214 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91fa5054220649a3815f536e03d9e2dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731255276016 2024-11-10T16:14:40,215 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 159dca58d917433591552859768845f4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:40,215 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 336986749d804c3886511687f3bc1c5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:40,215 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 33cd705eb5894299b061dc31ead4b2e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731255276016 2024-11-10T16:14:40,216 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 200ad0be388b452c8471f2f26c06abbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:40,226 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#260 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:40,226 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:40,226 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e0af21e1f71a429eb00905c9a7b3e38f is 50, key is test_row_0/A:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:40,227 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/4141925907154c5b806386cdd0b6f292 is 50, key is test_row_0/B:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742134_1310 (size=12915) 2024-11-10T16:14:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742133_1309 (size=12915) 2024-11-10T16:14:40,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,326 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:40,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:40,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/dd489345794d4ba4bd8ec14e2f23a692 is 50, key is test_row_0/A:col10/1731255279356/Put/seqid=0 2024-11-10T16:14:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742135_1311 (size=12301) 2024-11-10T16:14:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:40,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255340530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255340535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255340537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,643 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/4141925907154c5b806386cdd0b6f292 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4141925907154c5b806386cdd0b6f292 2024-11-10T16:14:40,643 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e0af21e1f71a429eb00905c9a7b3e38f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e0af21e1f71a429eb00905c9a7b3e38f 2024-11-10T16:14:40,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255340643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255340643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,649 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into 4141925907154c5b806386cdd0b6f292(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:40,649 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255280211; duration=0sec 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:40,649 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into e0af21e1f71a429eb00905c9a7b3e38f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:40,649 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255280211; duration=0sec 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:40,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:40,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:40,651 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:40,651 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:40,651 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:40,651 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/a27c34650a134244972ed5f0d752f570, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.2 K 2024-11-10T16:14:40,651 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a27c34650a134244972ed5f0d752f570, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255275354 2024-11-10T16:14:40,651 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e22b585fbac545649c9ff9b3fa969947, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731255276016 2024-11-10T16:14:40,652 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c6e4d3a12cb44c2a9ab4a629775f1807, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:40,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255340653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,658 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:40,659 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/189f1f2cd2ff4b4c8d34af9427ec3d15 is 50, key is test_row_0/C:col10/1731255279335/Put/seqid=0 2024-11-10T16:14:40,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742136_1312 (size=12915) 2024-11-10T16:14:40,739 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/dd489345794d4ba4bd8ec14e2f23a692 2024-11-10T16:14:40,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0df7504600a643aa8f9aa3ef18edf7dd is 50, key is test_row_0/B:col10/1731255279356/Put/seqid=0 2024-11-10T16:14:40,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742137_1313 (size=12301) 2024-11-10T16:14:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255340853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255340853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:40,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:40,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255340863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:41,068 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/189f1f2cd2ff4b4c8d34af9427ec3d15 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/189f1f2cd2ff4b4c8d34af9427ec3d15 2024-11-10T16:14:41,072 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into 189f1f2cd2ff4b4c8d34af9427ec3d15(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:41,072 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:41,072 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255280212; duration=0sec 2024-11-10T16:14:41,072 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:41,072 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:41,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0df7504600a643aa8f9aa3ef18edf7dd 2024-11-10T16:14:41,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b49d883d83dc4b3295ad63852f3f60e9 is 50, key is test_row_0/C:col10/1731255279356/Put/seqid=0 2024-11-10T16:14:41,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742138_1314 (size=12301) 2024-11-10T16:14:41,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255341164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,166 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255341164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255341173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,564 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b49d883d83dc4b3295ad63852f3f60e9 2024-11-10T16:14:41,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/dd489345794d4ba4bd8ec14e2f23a692 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692 2024-11-10T16:14:41,573 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T16:14:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/0df7504600a643aa8f9aa3ef18edf7dd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd 2024-11-10T16:14:41,578 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T16:14:41,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/b49d883d83dc4b3295ad63852f3f60e9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9 2024-11-10T16:14:41,583 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T16:14:41,584 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f8f59a82b84bc6f58d250116e5f29c9f in 1258ms, sequenceid=285, compaction requested=false 2024-11-10T16:14:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T16:14:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T16:14:41,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T16:14:41,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-10T16:14:41,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.7240 sec 2024-11-10T16:14:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:41,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:14:41,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:41,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:41,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/852e9d4cff48430ea8727d1041870700 is 50, key is test_row_0/A:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:41,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742139_1315 (size=12301) 2024-11-10T16:14:41,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/852e9d4cff48430ea8727d1041870700 2024-11-10T16:14:41,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/aa816c36544d4ef19c6be70929fd224d is 50, key is test_row_0/B:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:41,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742140_1316 (size=12301) 2024-11-10T16:14:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255341698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255341700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255341701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255341813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255341813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255341813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T16:14:42,013 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:42,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-10T16:14:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T16:14:42,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:42,016 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:42,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255342023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255342023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255342023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/aa816c36544d4ef19c6be70929fd224d 2024-11-10T16:14:42,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c087fa065c447b4a1a8f15d022219ab is 50, key is test_row_0/C:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:42,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742141_1317 (size=12301) 2024-11-10T16:14:42,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T16:14:42,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T16:14:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T16:14:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:42,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T16:14:42,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255342333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255342333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255342333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T16:14:42,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:42,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c087fa065c447b4a1a8f15d022219ab 2024-11-10T16:14:42,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/852e9d4cff48430ea8727d1041870700 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700 2024-11-10T16:14:42,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T16:14:42,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/aa816c36544d4ef19c6be70929fd224d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d 2024-11-10T16:14:42,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T16:14:42,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/0c087fa065c447b4a1a8f15d022219ab as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab 2024-11-10T16:14:42,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T16:14:42,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for f8f59a82b84bc6f58d250116e5f29c9f in 856ms, sequenceid=306, compaction requested=true 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:42,532 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:42,532 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:42,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:42,533 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:42,533 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:42,533 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:42,533 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:42,533 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,533 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,533 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4141925907154c5b806386cdd0b6f292, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.6 K 2024-11-10T16:14:42,533 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e0af21e1f71a429eb00905c9a7b3e38f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.6 K 2024-11-10T16:14:42,534 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0af21e1f71a429eb00905c9a7b3e38f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:42,534 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4141925907154c5b806386cdd0b6f292, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:42,534 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df7504600a643aa8f9aa3ef18edf7dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255279351 2024-11-10T16:14:42,534 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd489345794d4ba4bd8ec14e2f23a692, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255279351 2024-11-10T16:14:42,535 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852e9d4cff48430ea8727d1041870700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:42,535 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting aa816c36544d4ef19c6be70929fd224d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:42,542 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:42,543 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d is 50, key is test_row_0/B:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:42,543 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:42,544 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/648a177e6d6e4c858d0c9e57eaa9609b is 50, key is test_row_0/A:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:42,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742142_1318 (size=13017) 2024-11-10T16:14:42,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742143_1319 (size=13017) 2024-11-10T16:14:42,555 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/648a177e6d6e4c858d0c9e57eaa9609b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/648a177e6d6e4c858d0c9e57eaa9609b 2024-11-10T16:14:42,559 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d 2024-11-10T16:14:42,561 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 648a177e6d6e4c858d0c9e57eaa9609b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:42,561 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:42,561 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255282532; duration=0sec 2024-11-10T16:14:42,561 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:42,561 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:42,561 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:42,562 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:42,562 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:42,562 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,563 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/189f1f2cd2ff4b4c8d34af9427ec3d15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=36.6 K 2024-11-10T16:14:42,563 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 189f1f2cd2ff4b4c8d34af9427ec3d15, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731255278191 2024-11-10T16:14:42,563 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b49d883d83dc4b3295ad63852f3f60e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255279351 2024-11-10T16:14:42,564 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c087fa065c447b4a1a8f15d022219ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:42,566 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into c2c3ca7bd17f4bf5bcd12a50fe95af8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:42,566 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:42,566 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255282532; duration=0sec 2024-11-10T16:14:42,566 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:42,566 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:42,573 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:42,574 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/cd472019672e43e286de2a763f8b1812 is 50, key is test_row_0/C:col10/1731255280520/Put/seqid=0 2024-11-10T16:14:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742144_1320 (size=13017) 2024-11-10T16:14:42,586 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/cd472019672e43e286de2a763f8b1812 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/cd472019672e43e286de2a763f8b1812 2024-11-10T16:14:42,594 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into cd472019672e43e286de2a763f8b1812(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:42,594 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:42,594 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255282532; duration=0sec 2024-11-10T16:14:42,594 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:42,594 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:42,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-10T16:14:42,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:42,629 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:42,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:42,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/59677a1c976a47f09439245ec79ed8ea is 50, key is test_row_1/A:col10/1731255281700/Put/seqid=0 2024-11-10T16:14:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742145_1321 (size=9857) 2024-11-10T16:14:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T16:14:42,644 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/59677a1c976a47f09439245ec79ed8ea 2024-11-10T16:14:42,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5c45154e44d44c26a5c157bcb1a0c049 is 50, key is test_row_1/B:col10/1731255281700/Put/seqid=0 2024-11-10T16:14:42,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742146_1322 (size=9857) 2024-11-10T16:14:42,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5c45154e44d44c26a5c157bcb1a0c049 2024-11-10T16:14:42,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e08c193c94ea4afe9cd4f1852f789839 is 50, key is test_row_1/C:col10/1731255281700/Put/seqid=0 2024-11-10T16:14:42,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742147_1323 (size=9857) 2024-11-10T16:14:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:42,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. as already flushing 2024-11-10T16:14:42,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255342864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255342865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255342867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,928 INFO [Thread-1192 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1192. 2024-11-10T16:14:42,928 DEBUG [Thread-1192 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:42,928 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,928 DEBUG [Thread-1192 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,928 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:42,930 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-10T16:14:42,930 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:42,930 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,930 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,930 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:42,931 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-10T16:14:42,931 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:42,931 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,931 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,931 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:42,933 INFO [Thread-1194 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1194. 2024-11-10T16:14:42,933 DEBUG [Thread-1194 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:42,933 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,933 DEBUG [Thread-1194 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,933 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:42,934 INFO [Thread-1190 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1190. 2024-11-10T16:14:42,934 DEBUG [Thread-1190 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:42,934 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,934 DEBUG [Thread-1190 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:42,934 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49780 deadline: 1731255342973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49808 deadline: 1731255342973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49818 deadline: 1731255342973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:14:43,077 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e08c193c94ea4afe9cd4f1852f789839 2024-11-10T16:14:43,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/59677a1c976a47f09439245ec79ed8ea as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea 2024-11-10T16:14:43,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea, entries=100, sequenceid=323, filesize=9.6 K 2024-11-10T16:14:43,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/5c45154e44d44c26a5c157bcb1a0c049 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049 2024-11-10T16:14:43,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049, entries=100, sequenceid=323, filesize=9.6 K 2024-11-10T16:14:43,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/e08c193c94ea4afe9cd4f1852f789839 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839 2024-11-10T16:14:43,091 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839, entries=100, sequenceid=323, filesize=9.6 K 2024-11-10T16:14:43,092 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for f8f59a82b84bc6f58d250116e5f29c9f in 463ms, sequenceid=323, compaction requested=false 2024-11-10T16:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-10T16:14:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-10T16:14:43,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T16:14:43,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-10T16:14:43,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0810 sec 2024-11-10T16:14:43,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T16:14:43,154 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:43,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:43,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:14:43,185 INFO [Thread-1179 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1179. 2024-11-10T16:14:43,185 DEBUG [Thread-1179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:43,185 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-10T16:14:43,185 INFO [Thread-1181 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1181. 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:43,185 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:43,185 DEBUG [Thread-1181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:43,185 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 DEBUG [Thread-1179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:43,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:14:43,185 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 DEBUG [Thread-1181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:43,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:43,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:43,185 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:43,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e7ca2088d58c4b5c93c00163d7133606 is 50, key is test_row_0/A:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:43,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742148_1324 (size=12301) 2024-11-10T16:14:43,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e7ca2088d58c4b5c93c00163d7133606 2024-11-10T16:14:43,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/ee5b4a6a7bf24fe6bf7016c95b81edfd is 50, key is test_row_0/B:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:43,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742149_1325 (size=12301) 2024-11-10T16:14:44,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/ee5b4a6a7bf24fe6bf7016c95b81edfd 2024-11-10T16:14:44,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dcf76a705470474bb3f70e4ed86bc633 is 50, key is test_row_0/C:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:44,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742150_1326 (size=12301) 2024-11-10T16:14:44,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dcf76a705470474bb3f70e4ed86bc633 2024-11-10T16:14:44,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/e7ca2088d58c4b5c93c00163d7133606 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606 2024-11-10T16:14:44,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606, entries=150, sequenceid=344, filesize=12.0 K 2024-11-10T16:14:44,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/ee5b4a6a7bf24fe6bf7016c95b81edfd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd 2024-11-10T16:14:44,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd, entries=150, sequenceid=344, filesize=12.0 K 2024-11-10T16:14:44,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/dcf76a705470474bb3f70e4ed86bc633 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633 2024-11-10T16:14:44,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633, entries=150, sequenceid=344, filesize=12.0 K 2024-11-10T16:14:44,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for f8f59a82b84bc6f58d250116e5f29c9f in 1243ms, sequenceid=344, compaction requested=true 2024-11-10T16:14:44,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:44,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:44,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:44,427 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:44,427 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f8f59a82b84bc6f58d250116e5f29c9f:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:44,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:44,428 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:44,428 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:44,428 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:44,428 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/B is initiating minor compaction (all files) 2024-11-10T16:14:44,428 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/A is initiating minor compaction (all files) 2024-11-10T16:14:44,429 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/B in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:44,429 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/A in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:44,429 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/648a177e6d6e4c858d0c9e57eaa9609b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=34.4 K 2024-11-10T16:14:44,429 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=34.4 K 2024-11-10T16:14:44,429 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c2c3ca7bd17f4bf5bcd12a50fe95af8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:44,429 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 648a177e6d6e4c858d0c9e57eaa9609b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:44,429 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59677a1c976a47f09439245ec79ed8ea, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731255281693 2024-11-10T16:14:44,429 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c45154e44d44c26a5c157bcb1a0c049, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731255281693 2024-11-10T16:14:44,430 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7ca2088d58c4b5c93c00163d7133606, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731255282852 2024-11-10T16:14:44,430 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5b4a6a7bf24fe6bf7016c95b81edfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731255282852 2024-11-10T16:14:44,437 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:44,437 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#A#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:44,437 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/a1bc7e1874054db196eaef1ac03d7058 is 50, key is test_row_0/B:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:44,437 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/487ec91787364e73a984d09ea886d01d is 50, key is test_row_0/A:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742151_1327 (size=13119) 2024-11-10T16:14:44,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742152_1328 (size=13119) 2024-11-10T16:14:44,847 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/487ec91787364e73a984d09ea886d01d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/487ec91787364e73a984d09ea886d01d 2024-11-10T16:14:44,847 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/a1bc7e1874054db196eaef1ac03d7058 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a1bc7e1874054db196eaef1ac03d7058 2024-11-10T16:14:44,851 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/A of f8f59a82b84bc6f58d250116e5f29c9f into 487ec91787364e73a984d09ea886d01d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:44,851 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/B of f8f59a82b84bc6f58d250116e5f29c9f into a1bc7e1874054db196eaef1ac03d7058(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:44,851 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/A, priority=13, startTime=1731255284427; duration=0sec 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:44,851 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/B, priority=13, startTime=1731255284427; duration=0sec 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:A 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:44,851 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:B 2024-11-10T16:14:44,852 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:44,852 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): f8f59a82b84bc6f58d250116e5f29c9f/C is initiating minor compaction (all files) 2024-11-10T16:14:44,852 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f8f59a82b84bc6f58d250116e5f29c9f/C in TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:44,852 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/cd472019672e43e286de2a763f8b1812, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp, totalSize=34.4 K 2024-11-10T16:14:44,852 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd472019672e43e286de2a763f8b1812, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255280520 2024-11-10T16:14:44,852 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08c193c94ea4afe9cd4f1852f789839, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=323, earliestPutTs=1731255281693 2024-11-10T16:14:44,853 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcf76a705470474bb3f70e4ed86bc633, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731255282852 2024-11-10T16:14:44,860 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f8f59a82b84bc6f58d250116e5f29c9f#C#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:44,861 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/6ef73215dcd54f609fa34df79297f2e2 is 50, key is test_row_0/C:col10/1731255283183/Put/seqid=0 2024-11-10T16:14:44,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742153_1329 (size=13119) 2024-11-10T16:14:45,268 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/6ef73215dcd54f609fa34df79297f2e2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/6ef73215dcd54f609fa34df79297f2e2 2024-11-10T16:14:45,272 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f8f59a82b84bc6f58d250116e5f29c9f/C of f8f59a82b84bc6f58d250116e5f29c9f into 6ef73215dcd54f609fa34df79297f2e2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:45,272 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f8f59a82b84bc6f58d250116e5f29c9f: 2024-11-10T16:14:45,272 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., storeName=f8f59a82b84bc6f58d250116e5f29c9f/C, priority=13, startTime=1731255284428; duration=0sec 2024-11-10T16:14:45,272 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:45,272 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f8f59a82b84bc6f58d250116e5f29c9f:C 2024-11-10T16:14:45,804 INFO [Thread-1183 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1183. 2024-11-10T16:14:45,804 DEBUG [Thread-1183 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:45,804 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,804 DEBUG [Thread-1183 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:45,834 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-10T16:14:45,834 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:14:45,834 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,834 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3283 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3080 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3161 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3286 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3096 2024-11-10T16:14:45,835 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:14:45,835 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:14:45,835 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:14:45,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:45,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:14:45,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:14:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:45,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255285838"}]},"ts":"1731255285838"} 2024-11-10T16:14:45,840 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:14:45,840 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:14:45,840 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:14:45,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, UNASSIGN}] 2024-11-10T16:14:45,842 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, UNASSIGN 2024-11-10T16:14:45,842 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f8f59a82b84bc6f58d250116e5f29c9f, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:45,843 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, UNASSIGN because future has completed 2024-11-10T16:14:45,844 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:14:45,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:45,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:45,995 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing f8f59a82b84bc6f58d250116e5f29c9f, disabling compactions & flushes 2024-11-10T16:14:45,996 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. after waiting 0 ms 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:45,996 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing f8f59a82b84bc6f58d250116e5f29c9f 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=A 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=B 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f8f59a82b84bc6f58d250116e5f29c9f, store=C 2024-11-10T16:14:45,996 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:45,999 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/cd63815c0d4947c5898349d4013c0cea is 50, key is test_row_1/A:col10/1731255285833/Put/seqid=0 2024-11-10T16:14:46,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742154_1330 (size=9857) 2024-11-10T16:14:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:46,403 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/cd63815c0d4947c5898349d4013c0cea 2024-11-10T16:14:46,409 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/a7cf117ed73e490a8c5880fce126f8c5 is 50, key is test_row_1/B:col10/1731255285833/Put/seqid=0 2024-11-10T16:14:46,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742155_1331 (size=9857) 2024-11-10T16:14:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:46,813 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/a7cf117ed73e490a8c5880fce126f8c5 2024-11-10T16:14:46,819 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/410b1674e19e4c9cb3f319c825d13eb7 is 50, key is test_row_1/C:col10/1731255285833/Put/seqid=0 2024-11-10T16:14:46,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742156_1332 (size=9857) 2024-11-10T16:14:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:47,223 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/410b1674e19e4c9cb3f319c825d13eb7 2024-11-10T16:14:47,227 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/A/cd63815c0d4947c5898349d4013c0cea as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/cd63815c0d4947c5898349d4013c0cea 2024-11-10T16:14:47,230 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/cd63815c0d4947c5898349d4013c0cea, entries=100, sequenceid=352, filesize=9.6 K 2024-11-10T16:14:47,231 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/B/a7cf117ed73e490a8c5880fce126f8c5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a7cf117ed73e490a8c5880fce126f8c5 2024-11-10T16:14:47,234 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a7cf117ed73e490a8c5880fce126f8c5, entries=100, sequenceid=352, filesize=9.6 K 2024-11-10T16:14:47,234 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/.tmp/C/410b1674e19e4c9cb3f319c825d13eb7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/410b1674e19e4c9cb3f319c825d13eb7 2024-11-10T16:14:47,237 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/410b1674e19e4c9cb3f319c825d13eb7, entries=100, sequenceid=352, filesize=9.6 K 2024-11-10T16:14:47,238 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f8f59a82b84bc6f58d250116e5f29c9f in 1242ms, sequenceid=352, compaction requested=false 2024-11-10T16:14:47,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/84ede8172ce146c0a419b6091876228e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/19d8acea038e411a9668e135d9fd51f8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/4d643a85105a48429b3b36489e58606e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e48457e5351b4cb684c3592461896a58, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/3fe82f1ff09f4085a1b7d6b43693f000, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e0af21e1f71a429eb00905c9a7b3e38f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/648a177e6d6e4c858d0c9e57eaa9609b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606] to archive 2024-11-10T16:14:47,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:47,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/58cf7e96d51c4307a3ec0934793cd278 2024-11-10T16:14:47,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/14c7357b4c024af899f5fec3cccd593d 2024-11-10T16:14:47,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/84ede8172ce146c0a419b6091876228e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/84ede8172ce146c0a419b6091876228e 2024-11-10T16:14:47,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/6c2f42a6c12c416e829f61d0524c1ec7 2024-11-10T16:14:47,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/05688cbab9d348a8abf329cebab911be 2024-11-10T16:14:47,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/19d8acea038e411a9668e135d9fd51f8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/19d8acea038e411a9668e135d9fd51f8 2024-11-10T16:14:47,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e49cc42e600a4fc0a7b6e959bab0b872 2024-11-10T16:14:47,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b7d8627da21f447b882ca64097106e9f 2024-11-10T16:14:47,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/b013ac4eb13649dc8e162ae311ca5978 2024-11-10T16:14:47,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/4d643a85105a48429b3b36489e58606e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/4d643a85105a48429b3b36489e58606e 2024-11-10T16:14:47,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7a0f0884f2f946a798741f9ffaee2ee1 2024-11-10T16:14:47,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e48457e5351b4cb684c3592461896a58 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e48457e5351b4cb684c3592461896a58 2024-11-10T16:14:47,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/0adbe053e32a42fa8f2b0b9cf5529f30 2024-11-10T16:14:47,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/7ecfe1e1de0a4f32885cbc8cfcfd0d98 2024-11-10T16:14:47,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/3fe82f1ff09f4085a1b7d6b43693f000 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/3fe82f1ff09f4085a1b7d6b43693f000 2024-11-10T16:14:47,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/db0a706dbcf94498a12641dfea7bf4ab 2024-11-10T16:14:47,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/91fa5054220649a3815f536e03d9e2dd 2024-11-10T16:14:47,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e0af21e1f71a429eb00905c9a7b3e38f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e0af21e1f71a429eb00905c9a7b3e38f 2024-11-10T16:14:47,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/336986749d804c3886511687f3bc1c5a 2024-11-10T16:14:47,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/dd489345794d4ba4bd8ec14e2f23a692 2024-11-10T16:14:47,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/648a177e6d6e4c858d0c9e57eaa9609b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/648a177e6d6e4c858d0c9e57eaa9609b 2024-11-10T16:14:47,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/852e9d4cff48430ea8727d1041870700 2024-11-10T16:14:47,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/59677a1c976a47f09439245ec79ed8ea 2024-11-10T16:14:47,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/e7ca2088d58c4b5c93c00163d7133606 2024-11-10T16:14:47,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1ac4d31cf7014e378314171eb85dba30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/32cfbb066a464c3abec1b01aec27102f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4e59546c98c34ce2a01c5fa0720cad10, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/9f621dbee4a54ba08aac398b7a6d7723, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/159dca58d917433591552859768845f4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4141925907154c5b806386cdd0b6f292, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd] to archive 2024-11-10T16:14:47,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:47,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/53594484b0fa4cbabe86e6977e805f30 2024-11-10T16:14:47,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0329ee64ee0f4478b784a9c5107249c1 2024-11-10T16:14:47,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1ac4d31cf7014e378314171eb85dba30 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1ac4d31cf7014e378314171eb85dba30 2024-11-10T16:14:47,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6ba578df292b4fe5baf09f0b39cf375c 2024-11-10T16:14:47,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/914e0b1e6d0b44918a2a7e97a6b22287 2024-11-10T16:14:47,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/32cfbb066a464c3abec1b01aec27102f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/32cfbb066a464c3abec1b01aec27102f 2024-11-10T16:14:47,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/1fe4ff1793714f159350b1c47d791fda 2024-11-10T16:14:47,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/80491afe940149d1a5d7dd3498d9c8a9 2024-11-10T16:14:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4e59546c98c34ce2a01c5fa0720cad10 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4e59546c98c34ce2a01c5fa0720cad10 2024-11-10T16:14:47,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/86375dc4fa8142c49d6e0d4f399c1249 2024-11-10T16:14:47,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/683470be7614413da511a5d70a3741f2 2024-11-10T16:14:47,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/9f621dbee4a54ba08aac398b7a6d7723 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/9f621dbee4a54ba08aac398b7a6d7723 2024-11-10T16:14:47,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5b86a143cfb14dbb826c1c6c291925b5 2024-11-10T16:14:47,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/6b82dfa0723b490f83ad934c14897fc8 2024-11-10T16:14:47,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/159dca58d917433591552859768845f4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/159dca58d917433591552859768845f4 2024-11-10T16:14:47,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/69f37a62106b4cbcb25e406f84eda5c5 2024-11-10T16:14:47,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/33cd705eb5894299b061dc31ead4b2e6 2024-11-10T16:14:47,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4141925907154c5b806386cdd0b6f292 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/4141925907154c5b806386cdd0b6f292 2024-11-10T16:14:47,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/200ad0be388b452c8471f2f26c06abbd 2024-11-10T16:14:47,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/0df7504600a643aa8f9aa3ef18edf7dd 2024-11-10T16:14:47,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/c2c3ca7bd17f4bf5bcd12a50fe95af8d 2024-11-10T16:14:47,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/aa816c36544d4ef19c6be70929fd224d 2024-11-10T16:14:47,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/5c45154e44d44c26a5c157bcb1a0c049 2024-11-10T16:14:47,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/ee5b4a6a7bf24fe6bf7016c95b81edfd 2024-11-10T16:14:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fb67428753f442cbb2a4535046572b6c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/bddc3199ee4347a78bdb27e856251141, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/64987fb634044486aaf1c505dfb5392f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/79abd6fe1bfd48b3a09a590084422520, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/a27c34650a134244972ed5f0d752f570, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/189f1f2cd2ff4b4c8d34af9427ec3d15, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/cd472019672e43e286de2a763f8b1812, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633] to archive 2024-11-10T16:14:47,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:14:47,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/8d256aa14bfb43f4b4896d6372f91c04 2024-11-10T16:14:47,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b0385a0fd6c546308cc3c9f83896be10 2024-11-10T16:14:47,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fb67428753f442cbb2a4535046572b6c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fb67428753f442cbb2a4535046572b6c 2024-11-10T16:14:47,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/84039d2a9cda4e8da1c554d72ba11640 2024-11-10T16:14:47,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/fcad781cf9804af892c5a420dd8ec47d 2024-11-10T16:14:47,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/bddc3199ee4347a78bdb27e856251141 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/bddc3199ee4347a78bdb27e856251141 2024-11-10T16:14:47,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/7f74f35269f44dbdaaec34fc2462ebd2 2024-11-10T16:14:47,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c0f9023ab37458d9f418b349ae64893 2024-11-10T16:14:47,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/64987fb634044486aaf1c505dfb5392f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/64987fb634044486aaf1c505dfb5392f 2024-11-10T16:14:47,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dd26a8a84d654ccab3c6da5f25cc8cc2 2024-11-10T16:14:47,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/adc69eb164f440a895aef2aee7251f5b 2024-11-10T16:14:47,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/79abd6fe1bfd48b3a09a590084422520 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/79abd6fe1bfd48b3a09a590084422520 2024-11-10T16:14:47,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b81f268431bb4d748bbaa3d6c53ced16 2024-11-10T16:14:47,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/45aa596da4bf4fda96819e84d9be1348 2024-11-10T16:14:47,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/a27c34650a134244972ed5f0d752f570 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/a27c34650a134244972ed5f0d752f570 2024-11-10T16:14:47,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b5b174d08f7740f59c3baae79064d415 2024-11-10T16:14:47,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e22b585fbac545649c9ff9b3fa969947 2024-11-10T16:14:47,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/189f1f2cd2ff4b4c8d34af9427ec3d15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/189f1f2cd2ff4b4c8d34af9427ec3d15 2024-11-10T16:14:47,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/c6e4d3a12cb44c2a9ab4a629775f1807 2024-11-10T16:14:47,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/b49d883d83dc4b3295ad63852f3f60e9 2024-11-10T16:14:47,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/cd472019672e43e286de2a763f8b1812 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/cd472019672e43e286de2a763f8b1812 2024-11-10T16:14:47,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/0c087fa065c447b4a1a8f15d022219ab 2024-11-10T16:14:47,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/e08c193c94ea4afe9cd4f1852f789839 2024-11-10T16:14:47,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/dcf76a705470474bb3f70e4ed86bc633 2024-11-10T16:14:47,311 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/recovered.edits/355.seqid, newMaxSeqId=355, maxSeqId=1 2024-11-10T16:14:47,312 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. 2024-11-10T16:14:47,312 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for f8f59a82b84bc6f58d250116e5f29c9f: Waiting for close lock at 1731255285996Running coprocessor pre-close hooks at 1731255285996Disabling compacts and flushes for region at 1731255285996Disabling writes for close at 1731255285996Obtaining lock to block concurrent updates at 1731255285996Preparing flush snapshotting stores in f8f59a82b84bc6f58d250116e5f29c9f at 1731255285996Finished memstore snapshotting TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731255285996Flushing stores of TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f. at 1731255285997 (+1 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/A: creating writer at 1731255285997Flushing f8f59a82b84bc6f58d250116e5f29c9f/A: appending metadata at 1731255285999 (+2 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/A: closing flushed file at 1731255285999Flushing f8f59a82b84bc6f58d250116e5f29c9f/B: creating writer at 1731255286407 (+408 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/B: appending metadata at 1731255286409 (+2 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/B: closing flushed file at 1731255286409Flushing f8f59a82b84bc6f58d250116e5f29c9f/C: creating writer at 1731255286817 (+408 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/C: appending metadata at 1731255286819 (+2 ms)Flushing f8f59a82b84bc6f58d250116e5f29c9f/C: closing flushed file at 1731255286819Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67aba3ec: reopening flushed file at 1731255287227 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aafb448: reopening flushed file at 1731255287230 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c6c852d: reopening flushed file at 1731255287234 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for f8f59a82b84bc6f58d250116e5f29c9f in 1242ms, sequenceid=352, compaction requested=false at 1731255287238 (+4 ms)Writing region close event to WAL at 1731255287308 (+70 ms)Running coprocessor post-close hooks at 1731255287312 (+4 ms)Closed at 1731255287312 2024-11-10T16:14:47,314 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:47,314 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=f8f59a82b84bc6f58d250116e5f29c9f, regionState=CLOSED 2024-11-10T16:14:47,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:47,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-10T16:14:47,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure f8f59a82b84bc6f58d250116e5f29c9f, server=8e3de9160f9f,44105,1731255200149 in 1.4720 sec 2024-11-10T16:14:47,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-10T16:14:47,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f8f59a82b84bc6f58d250116e5f29c9f, UNASSIGN in 1.4770 sec 2024-11-10T16:14:47,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T16:14:47,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4800 sec 2024-11-10T16:14:47,323 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255287323"}]},"ts":"1731255287323"} 2024-11-10T16:14:47,324 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:14:47,324 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:14:47,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4880 sec 2024-11-10T16:14:47,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-10T16:14:47,983 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:47,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:14:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:47,986 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T16:14:47,986 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:47,989 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:47,991 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/recovered.edits] 2024-11-10T16:14:47,993 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/487ec91787364e73a984d09ea886d01d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/487ec91787364e73a984d09ea886d01d 2024-11-10T16:14:47,994 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/cd63815c0d4947c5898349d4013c0cea to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/A/cd63815c0d4947c5898349d4013c0cea 2024-11-10T16:14:47,995 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a1bc7e1874054db196eaef1ac03d7058 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a1bc7e1874054db196eaef1ac03d7058 2024-11-10T16:14:47,996 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a7cf117ed73e490a8c5880fce126f8c5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/B/a7cf117ed73e490a8c5880fce126f8c5 2024-11-10T16:14:47,998 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/410b1674e19e4c9cb3f319c825d13eb7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/410b1674e19e4c9cb3f319c825d13eb7 2024-11-10T16:14:47,999 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/6ef73215dcd54f609fa34df79297f2e2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/C/6ef73215dcd54f609fa34df79297f2e2 2024-11-10T16:14:48,001 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/recovered.edits/355.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f/recovered.edits/355.seqid 2024-11-10T16:14:48,002 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/f8f59a82b84bc6f58d250116e5f29c9f 2024-11-10T16:14:48,002 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:14:48,003 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:48,006 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:14:48,008 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:14:48,008 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:48,008 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:14:48,009 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255288008"}]},"ts":"9223372036854775807"} 2024-11-10T16:14:48,010 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:14:48,010 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f8f59a82b84bc6f58d250116e5f29c9f, NAME => 'TestAcidGuarantees,,1731255260737.f8f59a82b84bc6f58d250116e5f29c9f.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:14:48,010 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:14:48,010 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255288010"}]},"ts":"9223372036854775807"} 2024-11-10T16:14:48,012 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:14:48,012 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:48,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-10T16:14:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-10T16:14:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:14:48,094 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:48,103 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286154732_22 at /127.0.0.1:54926 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_75357217_22 at /127.0.0.1:38820 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1286154732_22 at /127.0.0.1:54922 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_75357217_22 at /127.0.0.1:34414 [Waiting for operation #197] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 345), ProcessCount=11 (was 11), AvailableMemoryMB=7060 (was 7130) 2024-11-10T16:14:48,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=342, ProcessCount=11, AvailableMemoryMB=7060 2024-11-10T16:14:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:14:48,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:14:48,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:48,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:14:48,115 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:48,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-10T16:14:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:48,115 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:14:48,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742157_1333 (size=1061) 2024-11-10T16:14:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:48,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T16:14:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:48,522 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:14:48,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742158_1334 (size=53) 2024-11-10T16:14:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:48,927 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:14:48,927 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 307cbe8008d263744dfb710286b3c81a, disabling compactions & flushes 2024-11-10T16:14:48,927 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:48,928 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:48,928 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. after waiting 0 ms 2024-11-10T16:14:48,928 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:48,928 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:48,928 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 307cbe8008d263744dfb710286b3c81a: Waiting for close lock at 1731255288927Disabling compacts and flushes for region at 1731255288927Disabling writes for close at 1731255288928 (+1 ms)Writing region close event to WAL at 1731255288928Closed at 1731255288928 2024-11-10T16:14:48,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:14:48,929 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255288929"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255288929"}]},"ts":"1731255288929"} 2024-11-10T16:14:48,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:14:48,931 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:14:48,931 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255288931"}]},"ts":"1731255288931"} 2024-11-10T16:14:48,932 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:14:48,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, ASSIGN}] 2024-11-10T16:14:48,934 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, ASSIGN 2024-11-10T16:14:48,934 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:14:49,085 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:49,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, ASSIGN because future has completed 2024-11-10T16:14:49,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:49,241 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:49,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:14:49,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:14:49,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,241 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,242 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,243 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:49,244 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName A 2024-11-10T16:14:49,244 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:49,244 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:49,244 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,245 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:49,245 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName B 2024-11-10T16:14:49,245 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:49,245 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:49,245 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,246 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:49,246 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName C 2024-11-10T16:14:49,246 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:49,247 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:49,247 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:49,247 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,247 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,247 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,248 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,248 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,249 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:14:49,249 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,251 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:14:49,251 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 307cbe8008d263744dfb710286b3c81a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61429814, jitterRate=-0.08462443947792053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:14:49,251 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:49,252 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 307cbe8008d263744dfb710286b3c81a: Running coprocessor pre-open hook at 1731255289241Writing region info on filesystem at 1731255289241Initializing all the Stores at 1731255289242 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255289242Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255289242Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255289242Cleaning up temporary data from old regions at 1731255289248 (+6 ms)Running coprocessor post-open hooks at 1731255289251 (+3 ms)Region opened successfully at 1731255289252 (+1 ms) 2024-11-10T16:14:49,253 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., pid=91, masterSystemTime=1731255289238 2024-11-10T16:14:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:49,254 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:49,254 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:49,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:49,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:49,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T16:14:49,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 in 169 msec 2024-11-10T16:14:49,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-10T16:14:49,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, ASSIGN in 325 msec 2024-11-10T16:14:49,260 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:14:49,260 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255289260"}]},"ts":"1731255289260"} 2024-11-10T16:14:49,261 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:14:49,262 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:14:49,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-10T16:14:50,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-10T16:14:50,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:50,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:50,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:50,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:50,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:50,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685dc8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:50,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:50,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6df4496f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:50,268 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:50,268 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:50,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:50,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:14:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:14:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38d98321 2024-11-10T16:14:50,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:14:50,271 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:14:50,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:14:50,273 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:14:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T16:14:50,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742159_1335 (size=1097) 2024-11-10T16:14:50,685 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T16:14:50,685 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T16:14:50,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:14:50,689 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE}] 2024-11-10T16:14:50,690 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE 2024-11-10T16:14:50,690 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:50,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE because future has completed 2024-11-10T16:14:50,692 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:14:50,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:50,844 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:50,844 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:14:50,844 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 307cbe8008d263744dfb710286b3c81a, disabling compactions & flushes 2024-11-10T16:14:50,844 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:50,844 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:50,844 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. after waiting 0 ms 2024-11-10T16:14:50,844 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:50,848 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T16:14:50,848 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:50,849 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 307cbe8008d263744dfb710286b3c81a: Waiting for close lock at 1731255290844Running coprocessor pre-close hooks at 1731255290844Disabling compacts and flushes for region at 1731255290844Disabling writes for close at 1731255290844Writing region close event to WAL at 1731255290845 (+1 ms)Running coprocessor post-close hooks at 1731255290848 (+3 ms)Closed at 1731255290848 2024-11-10T16:14:50,849 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 307cbe8008d263744dfb710286b3c81a to self. 2024-11-10T16:14:50,850 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:50,851 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=CLOSED 2024-11-10T16:14:50,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:50,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-10T16:14:50,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 in 161 msec 2024-11-10T16:14:50,855 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=true 2024-11-10T16:14:51,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE because future has completed 2024-11-10T16:14:51,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:14:51,161 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,162 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:14:51,162 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,162 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:14:51,162 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,162 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,163 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,164 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:51,164 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName A 2024-11-10T16:14:51,165 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:51,166 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:51,166 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,166 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:51,167 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName B 2024-11-10T16:14:51,167 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:51,167 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:51,167 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,168 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:14:51,168 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 307cbe8008d263744dfb710286b3c81a columnFamilyName C 2024-11-10T16:14:51,168 DEBUG [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:51,168 INFO [StoreOpener-307cbe8008d263744dfb710286b3c81a-1 {}] regionserver.HStore(327): Store=307cbe8008d263744dfb710286b3c81a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:14:51,168 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,168 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,169 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,170 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,171 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,171 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,171 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:14:51,172 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,173 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 307cbe8008d263744dfb710286b3c81a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68582209, jitterRate=0.021954551339149475}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:14:51,173 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,173 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 307cbe8008d263744dfb710286b3c81a: Running coprocessor pre-open hook at 1731255291162Writing region info on filesystem at 1731255291162Initializing all the Stores at 1731255291163 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255291163Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255291163Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255291163Cleaning up temporary data from old regions at 1731255291171 (+8 ms)Running coprocessor post-open hooks at 1731255291173 (+2 ms)Region opened successfully at 1731255291173 2024-11-10T16:14:51,174 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., pid=96, masterSystemTime=1731255291159 2024-11-10T16:14:51,175 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,175 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,177 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:14:51,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-10T16:14:51,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 in 171 msec 2024-11-10T16:14:51,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-10T16:14:51,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, REOPEN/MOVE in 490 msec 2024-11-10T16:14:51,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T16:14:51,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-10T16:14:51,185 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-10T16:14:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-10T16:14:51,186 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:51,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,188 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ada358c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,189 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714f92bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,191 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6607c337, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,192 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d94ec35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,194 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ba233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,195 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6b1082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,196 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce1fce2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,198 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f7c4f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,199 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,199 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b75da8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626ef79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,202 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b103af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3799027d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac09d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,206 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,207 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5921e67b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,209 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b13a7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,210 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d8ef43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,211 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50188, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ab121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:14:51,213 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:14:51,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c16db42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:14:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:14:51,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:14:51,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:14:51,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d839cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:14:51,215 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,215 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,215 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,215 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,216 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,216 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,216 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,216 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,217 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,218 DEBUG [Thread-1544 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,219 DEBUG [Thread-1544 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:51,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:51,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,220 DEBUG [Thread-1540 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,221 DEBUG [Thread-1540 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,221 DEBUG [Thread-1529 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:51,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:51,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,227 DEBUG [Thread-1531 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:14:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:14:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:14:51,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:14:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:14:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:14:51,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:51,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109062b001254c4720a346978fa91740d3_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255291235/Put/seqid=0 2024-11-10T16:14:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255351263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255351264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255351272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255351272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,276 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742160_1336 (size=17034) 2024-11-10T16:14:51,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255351273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:51,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255351383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255351384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255351384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255351384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255351393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255351593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255351593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255351593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255351593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255351613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,679 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:51,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,683 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109062b001254c4720a346978fa91740d3_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109062b001254c4720a346978fa91740d3_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:51,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/96f00c3067ad48ebb61c22e3333d3336, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:51,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/96f00c3067ad48ebb61c22e3333d3336 is 175, key is test_row_0/A:col10/1731255291235/Put/seqid=0 2024-11-10T16:14:51,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742161_1337 (size=48139) 2024-11-10T16:14:51,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:51,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255351913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255351913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255351913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255351914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:51,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255351933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:51,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:51,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:51,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:51,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,092 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/96f00c3067ad48ebb61c22e3333d3336 2024-11-10T16:14:52,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d1b8bce9a4924f6194369114e125df80 is 50, key is test_row_0/B:col10/1731255291235/Put/seqid=0 2024-11-10T16:14:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742162_1338 (size=12001) 2024-11-10T16:14:52,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:52,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255352423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255352424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:52,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255352433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255352433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:52,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:52,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255352453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:52,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d1b8bce9a4924f6194369114e125df80 2024-11-10T16:14:52,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/3180e5b012d547b8b116cd82117e8038 is 50, key is test_row_0/C:col10/1731255291235/Put/seqid=0 2024-11-10T16:14:52,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742163_1339 (size=12001) 2024-11-10T16:14:52,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,839 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:14:52,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:52,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:52,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:52,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/3180e5b012d547b8b116cd82117e8038 2024-11-10T16:14:52,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/96f00c3067ad48ebb61c22e3333d3336 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336 2024-11-10T16:14:52,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336, entries=250, sequenceid=17, filesize=47.0 K 2024-11-10T16:14:52,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d1b8bce9a4924f6194369114e125df80 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80 2024-11-10T16:14:52,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T16:14:52,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/3180e5b012d547b8b116cd82117e8038 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038 2024-11-10T16:14:52,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038, entries=150, sequenceid=17, filesize=11.7 K 2024-11-10T16:14:52,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 307cbe8008d263744dfb710286b3c81a in 1734ms, sequenceid=17, compaction requested=false 2024-11-10T16:14:52,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:53,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:53,062 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:14:53,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:53,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014f3307e361c4d36b9bc160531f369fa_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255291270/Put/seqid=0 2024-11-10T16:14:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742164_1340 (size=12154) 2024-11-10T16:14:53,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:53,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:53,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:53,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255353463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255353464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,478 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014f3307e361c4d36b9bc160531f369fa_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014f3307e361c4d36b9bc160531f369fa_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255353467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41972d6d0333455f911c98604fd2a7d7, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:53,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41972d6d0333455f911c98604fd2a7d7 is 175, key is test_row_0/A:col10/1731255291270/Put/seqid=0 2024-11-10T16:14:53,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742165_1341 (size=30955) 2024-11-10T16:14:53,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255353473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255353473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,486 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255353583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255353583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255353584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255353593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255353803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255353804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255353804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255353804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:53,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:53,884 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41972d6d0333455f911c98604fd2a7d7 2024-11-10T16:14:53,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/258f96a862414e439fc7368706ce90f9 is 50, key is test_row_0/B:col10/1731255291270/Put/seqid=0 2024-11-10T16:14:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742166_1342 (size=12001) 2024-11-10T16:14:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255354123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255354123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255354123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255354124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/258f96a862414e439fc7368706ce90f9 2024-11-10T16:14:54,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/1f3a478019ac4814a0f8e347fe9ee0db is 50, key is test_row_0/C:col10/1731255291270/Put/seqid=0 2024-11-10T16:14:54,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742167_1343 (size=12001) 2024-11-10T16:14:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255354643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255354644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255354644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255354644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:54,709 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/1f3a478019ac4814a0f8e347fe9ee0db 2024-11-10T16:14:54,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41972d6d0333455f911c98604fd2a7d7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7 2024-11-10T16:14:54,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T16:14:54,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/258f96a862414e439fc7368706ce90f9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9 2024-11-10T16:14:54,722 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T16:14:54,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/1f3a478019ac4814a0f8e347fe9ee0db as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db 2024-11-10T16:14:54,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T16:14:54,727 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 307cbe8008d263744dfb710286b3c81a in 1665ms, sequenceid=36, compaction requested=false 2024-11-10T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T16:14:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T16:14:54,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T16:14:54,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5080 sec 2024-11-10T16:14:54,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.5150 sec 2024-11-10T16:14:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T16:14:55,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:55,384 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T16:14:55,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:55,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:55,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:55,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:55,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:55,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:14:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:14:55,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:55,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:14:55,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:55,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:14:55,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:55,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103733e126cbdd448ba5ca28ff8ca5e4d4_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742168_1344 (size=14594) 2024-11-10T16:14:55,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:55,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:55,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255355650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255355664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255355664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255355664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,669 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,669 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255355673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,684 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:55,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:55,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255355773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,921 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:55,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103733e126cbdd448ba5ca28ff8ca5e4d4_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103733e126cbdd448ba5ca28ff8ca5e4d4_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:55,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:55,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 is 175, key is test_row_0/A:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742169_1345 (size=39549) 2024-11-10T16:14:55,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:55,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255355983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:55,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:55,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:55,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:55,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:56,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255356293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,332 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 2024-11-10T16:14:56,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3426c592860c4d7dbfdb1e359087376c is 50, key is test_row_0/B:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:56,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742170_1346 (size=12001) 2024-11-10T16:14:56,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:56,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3426c592860c4d7dbfdb1e359087376c 2024-11-10T16:14:56,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/647cbc6e84f9494d99b2311431089cfc is 50, key is test_row_0/C:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:56,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742171_1347 (size=12001) 2024-11-10T16:14:56,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255356813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:56,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:56,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:56,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:57,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:57,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/647cbc6e84f9494d99b2311431089cfc 2024-11-10T16:14:57,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 2024-11-10T16:14:57,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9, entries=200, sequenceid=50, filesize=38.6 K 2024-11-10T16:14:57,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3426c592860c4d7dbfdb1e359087376c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c 2024-11-10T16:14:57,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T16:14:57,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/647cbc6e84f9494d99b2311431089cfc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc 2024-11-10T16:14:57,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T16:14:57,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 307cbe8008d263744dfb710286b3c81a in 1664ms, sequenceid=50, compaction requested=true 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:57,174 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:14:57,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:57,174 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/A is initiating minor compaction (all files) 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/B is initiating minor compaction (all files) 2024-11-10T16:14:57,175 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/A in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,175 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/B in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,175 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=115.9 K 2024-11-10T16:14:57,175 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.2 K 2024-11-10T16:14:57,175 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9] 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b8bce9a4924f6194369114e125df80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731255291228 2024-11-10T16:14:57,175 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f00c3067ad48ebb61c22e3333d3336, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731255291228 2024-11-10T16:14:57,176 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 258f96a862414e439fc7368706ce90f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731255291262 2024-11-10T16:14:57,176 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41972d6d0333455f911c98604fd2a7d7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731255291262 2024-11-10T16:14:57,176 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3426c592860c4d7dbfdb1e359087376c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293463 2024-11-10T16:14:57,176 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1df3be97f6d4b5cbc26b1fdf12f0fa9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293449 2024-11-10T16:14:57,181 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:57,183 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110124a0a88de664bb891231065ff99bc49_307cbe8008d263744dfb710286b3c81a store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:57,183 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#B#compaction#294 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:57,184 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/b8209d982fe2463e9a1bb65b28f26528 is 50, key is test_row_0/B:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:57,185 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110124a0a88de664bb891231065ff99bc49_307cbe8008d263744dfb710286b3c81a, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:57,185 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110124a0a88de664bb891231065ff99bc49_307cbe8008d263744dfb710286b3c81a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:57,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742172_1348 (size=12104) 2024-11-10T16:14:57,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742173_1349 (size=4469) 2024-11-10T16:14:57,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T16:14:57,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,224 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:14:57,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:57,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110464f44f9f06d4804ae997b8e8b846019_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255295598/Put/seqid=0 2024-11-10T16:14:57,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742174_1350 (size=12154) 2024-11-10T16:14:57,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:57,239 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110464f44f9f06d4804ae997b8e8b846019_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110464f44f9f06d4804ae997b8e8b846019_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a00143ff8f7b4da8a10a564537ab70c2, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:57,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a00143ff8f7b4da8a10a564537ab70c2 is 175, key is test_row_0/A:col10/1731255295598/Put/seqid=0 2024-11-10T16:14:57,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742175_1351 (size=30955) 2024-11-10T16:14:57,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:57,592 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/b8209d982fe2463e9a1bb65b28f26528 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/b8209d982fe2463e9a1bb65b28f26528 2024-11-10T16:14:57,594 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#A#compaction#293 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:57,595 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/20563eda1b6f4fa882ba726f2c4c740e is 175, key is test_row_0/A:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:57,598 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/B of 307cbe8008d263744dfb710286b3c81a into b8209d982fe2463e9a1bb65b28f26528(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:57,598 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:57,598 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/B, priority=13, startTime=1731255297174; duration=0sec 2024-11-10T16:14:57,598 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:14:57,598 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:14:57,598 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:14:57,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742176_1352 (size=31058) 2024-11-10T16:14:57,601 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:14:57,602 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/C is initiating minor compaction (all files) 2024-11-10T16:14:57,602 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/C in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:57,602 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.2 K 2024-11-10T16:14:57,602 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3180e5b012d547b8b116cd82117e8038, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731255291228 2024-11-10T16:14:57,602 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3a478019ac4814a0f8e347fe9ee0db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731255291262 2024-11-10T16:14:57,605 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 647cbc6e84f9494d99b2311431089cfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293463 2024-11-10T16:14:57,611 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:14:57,612 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/4b15e48ae9b84d63ae4d896ea6dcb4e3 is 50, key is test_row_0/C:col10/1731255295508/Put/seqid=0 2024-11-10T16:14:57,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742177_1353 (size=12104) 2024-11-10T16:14:57,645 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a00143ff8f7b4da8a10a564537ab70c2 2024-11-10T16:14:57,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/751a4c1de7254b9eb0731fbc550ffeba is 50, key is test_row_0/B:col10/1731255295598/Put/seqid=0 2024-11-10T16:14:57,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742178_1354 (size=12001) 2024-11-10T16:14:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:57,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:57,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255357704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255357709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255357718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255357719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255357823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255357823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255357833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255357833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,843 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:57,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255357843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:57,852 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,006 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/20563eda1b6f4fa882ba726f2c4c740e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e 2024-11-10T16:14:58,011 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/A of 307cbe8008d263744dfb710286b3c81a into 20563eda1b6f4fa882ba726f2c4c740e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:58,011 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:58,011 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/A, priority=13, startTime=1731255297174; duration=0sec 2024-11-10T16:14:58,011 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:58,011 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:14:58,022 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/4b15e48ae9b84d63ae4d896ea6dcb4e3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/4b15e48ae9b84d63ae4d896ea6dcb4e3 2024-11-10T16:14:58,026 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/C of 307cbe8008d263744dfb710286b3c81a into 4b15e48ae9b84d63ae4d896ea6dcb4e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:14:58,026 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:58,026 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/C, priority=13, startTime=1731255297174; duration=0sec 2024-11-10T16:14:58,026 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:14:58,026 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:14:58,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255358033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,041 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255358033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/751a4c1de7254b9eb0731fbc550ffeba 2024-11-10T16:14:58,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255358053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/d7e501be792644a4b7701d3fd9f506b9 is 50, key is test_row_0/C:col10/1731255295598/Put/seqid=0 2024-11-10T16:14:58,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742179_1355 (size=12001) 2024-11-10T16:14:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255358063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255358353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255358353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255358363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255358383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,467 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/d7e501be792644a4b7701d3fd9f506b9 2024-11-10T16:14:58,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a00143ff8f7b4da8a10a564537ab70c2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2 2024-11-10T16:14:58,475 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2, entries=150, sequenceid=68, filesize=30.2 K 2024-11-10T16:14:58,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/751a4c1de7254b9eb0731fbc550ffeba as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba 2024-11-10T16:14:58,479 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T16:14:58,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/d7e501be792644a4b7701d3fd9f506b9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9 2024-11-10T16:14:58,483 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9, entries=150, sequenceid=68, filesize=11.7 K 2024-11-10T16:14:58,484 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 307cbe8008d263744dfb710286b3c81a in 1260ms, sequenceid=68, compaction requested=false 2024-11-10T16:14:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:14:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T16:14:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T16:14:58,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T16:14:58,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0980 sec 2024-11-10T16:14:58,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.1040 sec 2024-11-10T16:14:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:58,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:14:58,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:14:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:14:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:14:58,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:14:58,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110279d0136fac2454e9e4d50558c47d4a8_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255297718/Put/seqid=0 2024-11-10T16:14:58,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742180_1356 (size=14594) 2024-11-10T16:14:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255358913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255358914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255358920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:58,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255358922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255359033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255359033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255359033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255359043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255359253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255359253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255359253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255359264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,283 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:14:59,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110279d0136fac2454e9e4d50558c47d4a8_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110279d0136fac2454e9e4d50558c47d4a8_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:14:59,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/570a90f530ad450496b5746341b171c9, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:14:59,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/570a90f530ad450496b5746341b171c9 is 175, key is test_row_0/A:col10/1731255297718/Put/seqid=0 2024-11-10T16:14:59,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742181_1357 (size=39549) 2024-11-10T16:14:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T16:14:59,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:14:59,543 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:14:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-10T16:14:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:14:59,545 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:14:59,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:14:59,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:14:59,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255359573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255359573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255359573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255359583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:14:59,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:14:59,692 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/570a90f530ad450496b5746341b171c9 2024-11-10T16:14:59,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T16:14:59,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:59,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/bae14bc4e3f74139b263787fc1196fb8 is 50, key is test_row_0/B:col10/1731255297718/Put/seqid=0 2024-11-10T16:14:59,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742182_1358 (size=12001) 2024-11-10T16:14:59,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T16:14:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:14:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:14:59,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:14:59,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:14:59,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:14:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255359863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:14:59,878 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:14:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:14:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:00,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T16:15:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:00,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255360093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255360094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255360094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:00,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255360094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:00,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:00,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/bae14bc4e3f74139b263787fc1196fb8 2024-11-10T16:15:00,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/55068b97a974442eb21fe7af11688f83 is 50, key is test_row_0/C:col10/1731255297718/Put/seqid=0 2024-11-10T16:15:00,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742183_1359 (size=12001) 2024-11-10T16:15:00,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/55068b97a974442eb21fe7af11688f83 2024-11-10T16:15:00,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/570a90f530ad450496b5746341b171c9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9 2024-11-10T16:15:00,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9, entries=200, sequenceid=86, filesize=38.6 K 2024-11-10T16:15:00,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/bae14bc4e3f74139b263787fc1196fb8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8 2024-11-10T16:15:00,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T16:15:00,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/55068b97a974442eb21fe7af11688f83 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83 2024-11-10T16:15:00,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T16:15:00,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 307cbe8008d263744dfb710286b3c81a in 1270ms, sequenceid=86, compaction requested=true 2024-11-10T16:15:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:00,139 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:00,139 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:00,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:00,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:00,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:00,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:00,140 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:00,140 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/B is initiating minor compaction (all files) 2024-11-10T16:15:00,140 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/B in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,141 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/b8209d982fe2463e9a1bb65b28f26528, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.3 K 2024-11-10T16:15:00,141 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b8209d982fe2463e9a1bb65b28f26528, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293463 2024-11-10T16:15:00,141 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 751a4c1de7254b9eb0731fbc550ffeba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731255295598 2024-11-10T16:15:00,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:00,142 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting bae14bc4e3f74139b263787fc1196fb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:00,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/A is initiating minor compaction (all files) 2024-11-10T16:15:00,142 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/A in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,142 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=99.2 K 2024-11-10T16:15:00,142 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9] 2024-11-10T16:15:00,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20563eda1b6f4fa882ba726f2c4c740e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293463 2024-11-10T16:15:00,143 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a00143ff8f7b4da8a10a564537ab70c2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731255295598 2024-11-10T16:15:00,143 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 570a90f530ad450496b5746341b171c9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:00,155 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#B#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:00,156 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/a7d9d51f1f21404f839c8ac51fe1876c is 50, key is test_row_0/B:col10/1731255297718/Put/seqid=0 2024-11-10T16:15:00,157 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:00,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T16:15:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:15:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:00,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:00,169 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110f9cfcaf37d1d4a6c814b697d6fef1064_307cbe8008d263744dfb710286b3c81a store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:00,171 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110f9cfcaf37d1d4a6c814b697d6fef1064_307cbe8008d263744dfb710286b3c81a, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:00,172 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f9cfcaf37d1d4a6c814b697d6fef1064_307cbe8008d263744dfb710286b3c81a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:00,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:15:00,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742184_1360 (size=12207) 2024-11-10T16:15:00,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fa3108d68088489ba7daaea5e0720c46_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255298913/Put/seqid=0 2024-11-10T16:15:00,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742185_1361 (size=4469) 2024-11-10T16:15:00,187 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/a7d9d51f1f21404f839c8ac51fe1876c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/a7d9d51f1f21404f839c8ac51fe1876c 2024-11-10T16:15:00,193 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/B of 307cbe8008d263744dfb710286b3c81a into a7d9d51f1f21404f839c8ac51fe1876c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:00,193 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:00,194 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/B, priority=13, startTime=1731255300139; duration=0sec 2024-11-10T16:15:00,194 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:00,194 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:15:00,194 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:00,194 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:00,195 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/C is initiating minor compaction (all files) 2024-11-10T16:15:00,195 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/C in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:00,195 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/4b15e48ae9b84d63ae4d896ea6dcb4e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.3 K 2024-11-10T16:15:00,195 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b15e48ae9b84d63ae4d896ea6dcb4e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731255293463 2024-11-10T16:15:00,195 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e501be792644a4b7701d3fd9f506b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731255295598 2024-11-10T16:15:00,195 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 55068b97a974442eb21fe7af11688f83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742186_1362 (size=12154) 2024-11-10T16:15:00,206 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:00,206 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/a8dbf97fc85d4630b5e1d6a5a90d2122 is 50, key is test_row_0/C:col10/1731255297718/Put/seqid=0 2024-11-10T16:15:00,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742187_1363 (size=12207) 2024-11-10T16:15:00,586 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#A#compaction#303 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:00,586 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/22b26b0dc0104d76a0e528863b706ad2 is 175, key is test_row_0/A:col10/1731255297718/Put/seqid=0 2024-11-10T16:15:00,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742188_1364 (size=31161) 2024-11-10T16:15:00,595 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/22b26b0dc0104d76a0e528863b706ad2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2 2024-11-10T16:15:00,600 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/A of 307cbe8008d263744dfb710286b3c81a into 22b26b0dc0104d76a0e528863b706ad2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:00,600 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:00,600 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/A, priority=13, startTime=1731255300139; duration=0sec 2024-11-10T16:15:00,600 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:00,600 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:15:00,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:00,607 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110fa3108d68088489ba7daaea5e0720c46_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fa3108d68088489ba7daaea5e0720c46_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:00,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/6d5e118d705f4aeb976b50deabfc98b3, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:00,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/6d5e118d705f4aeb976b50deabfc98b3 is 175, key is test_row_0/A:col10/1731255298913/Put/seqid=0 2024-11-10T16:15:00,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742189_1365 (size=30955) 2024-11-10T16:15:00,622 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/a8dbf97fc85d4630b5e1d6a5a90d2122 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/a8dbf97fc85d4630b5e1d6a5a90d2122 2024-11-10T16:15:00,627 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/C of 307cbe8008d263744dfb710286b3c81a into a8dbf97fc85d4630b5e1d6a5a90d2122(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:00,627 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:00,627 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/C, priority=13, startTime=1731255300140; duration=0sec 2024-11-10T16:15:00,627 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:00,628 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:15:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:15:01,013 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/6d5e118d705f4aeb976b50deabfc98b3 2024-11-10T16:15:01,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/e8eb03125a94466e97e5a37868466059 is 50, key is test_row_0/B:col10/1731255298913/Put/seqid=0 2024-11-10T16:15:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742190_1366 (size=12001) 2024-11-10T16:15:01,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:01,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255361136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255361135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255361137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255361151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255361253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255361253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255361253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255361263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,426 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/e8eb03125a94466e97e5a37868466059 2024-11-10T16:15:01,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/7b8f402544454deea5f9c2b540210fc9 is 50, key is test_row_0/C:col10/1731255298913/Put/seqid=0 2024-11-10T16:15:01,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742191_1367 (size=12001) 2024-11-10T16:15:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255361463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255361463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255361463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255361473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:15:01,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255361773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:01,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255361783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:01,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/7b8f402544454deea5f9c2b540210fc9 2024-11-10T16:15:01,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/6d5e118d705f4aeb976b50deabfc98b3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3 2024-11-10T16:15:01,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3, entries=150, sequenceid=103, filesize=30.2 K 2024-11-10T16:15:01,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/e8eb03125a94466e97e5a37868466059 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059 2024-11-10T16:15:01,852 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T16:15:01,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/7b8f402544454deea5f9c2b540210fc9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9 2024-11-10T16:15:01,857 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T16:15:01,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 307cbe8008d263744dfb710286b3c81a in 1700ms, sequenceid=103, compaction requested=false 2024-11-10T16:15:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-10T16:15:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-10T16:15:01,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T16:15:01,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3130 sec 2024-11-10T16:15:01,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3180 sec 2024-11-10T16:15:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:02,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:15:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:02,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:02,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103819b7e2534840b59f9ad9127744bff9_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:02,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742192_1368 (size=17034) 2024-11-10T16:15:02,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255362320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255362324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255362325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255362331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255362444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255362444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255362444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255362453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255362664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255362664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255362664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255362664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:02,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103819b7e2534840b59f9ad9127744bff9_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103819b7e2534840b59f9ad9127744bff9_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:02,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a7aaf96deca74655987c9b2ccfc25027, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:02,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a7aaf96deca74655987c9b2ccfc25027 is 175, key is test_row_0/A:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:02,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742193_1369 (size=48139) 2024-11-10T16:15:02,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255362983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255362983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255362983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255362983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:02,989 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:03,118 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a7aaf96deca74655987c9b2ccfc25027 2024-11-10T16:15:03,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/51a55332752d4bd3a4c8b726c615cfa3 is 50, key is test_row_0/B:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:03,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742194_1370 (size=12001) 2024-11-10T16:15:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255363493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:03,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:03,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255363493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:03,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:03,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255363503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255363503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:03,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:03,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/51a55332752d4bd3a4c8b726c615cfa3 2024-11-10T16:15:03,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/08716ad9483c40efaaf4c620579b610f is 50, key is test_row_0/C:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:03,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742195_1371 (size=12001) 2024-11-10T16:15:03,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/08716ad9483c40efaaf4c620579b610f 2024-11-10T16:15:03,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/a7aaf96deca74655987c9b2ccfc25027 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027 2024-11-10T16:15:03,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027, entries=250, sequenceid=123, filesize=47.0 K 2024-11-10T16:15:03,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/51a55332752d4bd3a4c8b726c615cfa3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3 2024-11-10T16:15:03,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T16:15:03,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/08716ad9483c40efaaf4c620579b610f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f 2024-11-10T16:15:03,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T16:15:03,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 307cbe8008d263744dfb710286b3c81a in 1274ms, sequenceid=123, compaction requested=true 2024-11-10T16:15:03,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:03,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:03,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:03,571 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:03,572 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:03,572 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:03,572 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/B is initiating minor compaction (all files) 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/A is initiating minor compaction (all files) 2024-11-10T16:15:03,573 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/B in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:03,573 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/A in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:03,573 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/a7d9d51f1f21404f839c8ac51fe1876c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.4 K 2024-11-10T16:15:03,573 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=107.7 K 2024-11-10T16:15:03,573 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027] 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a7d9d51f1f21404f839c8ac51fe1876c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22b26b0dc0104d76a0e528863b706ad2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:03,573 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e8eb03125a94466e97e5a37868466059, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731255298912 2024-11-10T16:15:03,574 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a55332752d4bd3a4c8b726c615cfa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:03,574 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d5e118d705f4aeb976b50deabfc98b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731255298912 2024-11-10T16:15:03,574 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7aaf96deca74655987c9b2ccfc25027, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:03,582 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#B#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:03,583 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/257c0b941a2e4b5b969251b50f3a622d is 50, key is test_row_0/B:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:03,587 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:03,590 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a94fcc0c01dd4e08b0252388b383768a_307cbe8008d263744dfb710286b3c81a store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:03,591 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a94fcc0c01dd4e08b0252388b383768a_307cbe8008d263744dfb710286b3c81a, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:03,591 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a94fcc0c01dd4e08b0252388b383768a_307cbe8008d263744dfb710286b3c81a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:03,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742196_1372 (size=12309) 2024-11-10T16:15:03,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742197_1373 (size=4469) 2024-11-10T16:15:03,601 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#A#compaction#312 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:03,602 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/8cd8aae977d2420190478d612ac11bcc is 175, key is test_row_0/A:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:03,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742198_1374 (size=31263) 2024-11-10T16:15:03,636 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/8cd8aae977d2420190478d612ac11bcc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc 2024-11-10T16:15:03,649 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/A of 307cbe8008d263744dfb710286b3c81a into 8cd8aae977d2420190478d612ac11bcc(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:03,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:03,649 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/A, priority=13, startTime=1731255303571; duration=0sec 2024-11-10T16:15:03,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:03,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:15:03,649 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:03,651 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:03,651 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/C is initiating minor compaction (all files) 2024-11-10T16:15:03,651 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/C in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:03,651 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/a8dbf97fc85d4630b5e1d6a5a90d2122, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.4 K 2024-11-10T16:15:03,651 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8dbf97fc85d4630b5e1d6a5a90d2122, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255297701 2024-11-10T16:15:03,652 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b8f402544454deea5f9c2b540210fc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731255298912 2024-11-10T16:15:03,652 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08716ad9483c40efaaf4c620579b610f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:03,660 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#C#compaction#313 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:03,660 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/5b06d49cca284d89ab490b1bb6e69a01 is 50, key is test_row_0/C:col10/1731255301149/Put/seqid=0 2024-11-10T16:15:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742199_1375 (size=12309) 2024-11-10T16:15:03,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T16:15:03,714 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:03,714 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:03,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T16:15:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:03,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:03,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:03,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:03,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:03,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T16:15:03,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:03,869 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:03,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:03,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109556038b7ec244d2ae3a0b8e16a3a44e_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255302329/Put/seqid=0 2024-11-10T16:15:03,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742200_1376 (size=12304) 2024-11-10T16:15:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:03,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:03,997 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/257c0b941a2e4b5b969251b50f3a622d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/257c0b941a2e4b5b969251b50f3a622d 2024-11-10T16:15:04,001 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/B of 307cbe8008d263744dfb710286b3c81a into 257c0b941a2e4b5b969251b50f3a622d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:04,001 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:04,001 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/B, priority=13, startTime=1731255303571; duration=0sec 2024-11-10T16:15:04,001 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:04,001 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:15:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:04,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255364031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,069 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/5b06d49cca284d89ab490b1bb6e69a01 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/5b06d49cca284d89ab490b1bb6e69a01 2024-11-10T16:15:04,074 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/C of 307cbe8008d263744dfb710286b3c81a into 5b06d49cca284d89ab490b1bb6e69a01(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:04,074 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:04,074 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/C, priority=13, startTime=1731255303572; duration=0sec 2024-11-10T16:15:04,074 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:04,074 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:15:04,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255364153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:04,290 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109556038b7ec244d2ae3a0b8e16a3a44e_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109556038b7ec244d2ae3a0b8e16a3a44e_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:04,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/fd1caa5f825240ccaf0e74ea4e4e7b87, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:04,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/fd1caa5f825240ccaf0e74ea4e4e7b87 is 175, key is test_row_0/A:col10/1731255302329/Put/seqid=0 2024-11-10T16:15:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742201_1377 (size=31105) 2024-11-10T16:15:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:04,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255364363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255364513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255364513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,525 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:04,525 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255364523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255364523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,530 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,530 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,530 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 307cbe8008d263744dfb710286b3c81a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255364683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:04,697 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/fd1caa5f825240ccaf0e74ea4e4e7b87 2024-11-10T16:15:04,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3af137dc0c8e435487ef33ec41227a08 is 50, key is test_row_0/B:col10/1731255302329/Put/seqid=0 2024-11-10T16:15:04,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742202_1378 (size=12151) 2024-11-10T16:15:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:05,108 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3af137dc0c8e435487ef33ec41227a08 2024-11-10T16:15:05,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/598e521a0c4b4f6e99a3761ca695e15b is 50, key is test_row_0/C:col10/1731255302329/Put/seqid=0 2024-11-10T16:15:05,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742203_1379 (size=12151) 2024-11-10T16:15:05,119 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/598e521a0c4b4f6e99a3761ca695e15b 2024-11-10T16:15:05,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/fd1caa5f825240ccaf0e74ea4e4e7b87 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87 2024-11-10T16:15:05,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87, entries=150, sequenceid=139, filesize=30.4 K 2024-11-10T16:15:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/3af137dc0c8e435487ef33ec41227a08 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08 2024-11-10T16:15:05,132 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T16:15:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/598e521a0c4b4f6e99a3761ca695e15b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b 2024-11-10T16:15:05,136 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T16:15:05,137 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 307cbe8008d263744dfb710286b3c81a in 1268ms, sequenceid=139, compaction requested=false 2024-11-10T16:15:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T16:15:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T16:15:05,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T16:15:05,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-11-10T16:15:05,143 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.4270 sec 2024-11-10T16:15:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:05,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:05,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:05,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111008381ac802e046ce8807cc1f782f3b1d_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:05,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742204_1380 (size=14794) 2024-11-10T16:15:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255365287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:05,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255365403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:05,608 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:05,612 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111008381ac802e046ce8807cc1f782f3b1d_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111008381ac802e046ce8807cc1f782f3b1d_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:05,613 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/313901096ef44d6fb97a72353f0b1187, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:05,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/313901096ef44d6fb97a72353f0b1187 is 175, key is test_row_0/A:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:05,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742205_1381 (size=39749) 2024-11-10T16:15:05,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255365623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T16:15:05,863 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:05,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T16:15:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:05,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:05,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:05,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255365943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:06,017 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/313901096ef44d6fb97a72353f0b1187 2024-11-10T16:15:06,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/4d090cde54844432bb04c1d59841045c is 50, key is test_row_0/B:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:06,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742206_1382 (size=12151) 2024-11-10T16:15:06,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:06,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/4d090cde54844432bb04c1d59841045c 2024-11-10T16:15:06,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/923e4c049e904f50a4f0716fd90f1744 is 50, key is test_row_0/C:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742207_1383 (size=12151) 2024-11-10T16:15:06,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/923e4c049e904f50a4f0716fd90f1744 2024-11-10T16:15:06,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/313901096ef44d6fb97a72353f0b1187 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187 2024-11-10T16:15:06,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187, entries=200, sequenceid=159, filesize=38.8 K 2024-11-10T16:15:06,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/4d090cde54844432bb04c1d59841045c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c 2024-11-10T16:15:06,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T16:15:06,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/923e4c049e904f50a4f0716fd90f1744 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744 2024-11-10T16:15:06,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T16:15:06,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 307cbe8008d263744dfb710286b3c81a in 1267ms, sequenceid=159, compaction requested=true 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:06,465 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:06,465 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/A is initiating minor compaction (all files) 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/B is initiating minor compaction (all files) 2024-11-10T16:15:06,466 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/A in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,466 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/B in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,466 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=99.7 K 2024-11-10T16:15:06,466 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/257c0b941a2e4b5b969251b50f3a622d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.8 K 2024-11-10T16:15:06,466 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187] 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 257c0b941a2e4b5b969251b50f3a622d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:06,466 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cd8aae977d2420190478d612ac11bcc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:06,467 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1caa5f825240ccaf0e74ea4e4e7b87, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255302315 2024-11-10T16:15:06,467 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af137dc0c8e435487ef33ec41227a08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255302315 2024-11-10T16:15:06,468 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313901096ef44d6fb97a72353f0b1187, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:06,468 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d090cde54844432bb04c1d59841045c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:06,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:06,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:06,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:06,476 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:06,478 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:06,478 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111053537461560544e09b350914a469144c_307cbe8008d263744dfb710286b3c81a store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:06,478 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/634a4e73ebf34d85825f168d2fd0fbbb is 50, key is test_row_0/B:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:06,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026766b27809a4ed3a80c096e95fa4788_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255305285/Put/seqid=0 2024-11-10T16:15:06,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,480 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111053537461560544e09b350914a469144c_307cbe8008d263744dfb710286b3c81a, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:06,480 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111053537461560544e09b350914a469144c_307cbe8008d263744dfb710286b3c81a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:06,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742208_1384 (size=14794) 2024-11-10T16:15:06,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:06,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742209_1385 (size=12561) 2024-11-10T16:15:06,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:06,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742210_1386 (size=4469) 2024-11-10T16:15:06,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111026766b27809a4ed3a80c096e95fa4788_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026766b27809a4ed3a80c096e95fa4788_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:06,497 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#A#compaction#321 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:06,498 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/1fa3516aa4d94882a4bfe0018f10e4fd is 175, key is test_row_0/A:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:06,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/11eb92d1ac7f4265b79c5030bea74e6a, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:06,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/11eb92d1ac7f4265b79c5030bea74e6a is 175, key is test_row_0/A:col10/1731255305285/Put/seqid=0 2024-11-10T16:15:06,508 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/634a4e73ebf34d85825f168d2fd0fbbb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/634a4e73ebf34d85825f168d2fd0fbbb 2024-11-10T16:15:06,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742211_1387 (size=31515) 2024-11-10T16:15:06,514 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/B of 307cbe8008d263744dfb710286b3c81a into 634a4e73ebf34d85825f168d2fd0fbbb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:06,514 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:06,514 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/B, priority=13, startTime=1731255306465; duration=0sec 2024-11-10T16:15:06,514 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:06,514 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:15:06,514 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:06,516 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:06,516 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/C is initiating minor compaction (all files) 2024-11-10T16:15:06,516 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/C in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,517 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/5b06d49cca284d89ab490b1bb6e69a01, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=35.8 K 2024-11-10T16:15:06,517 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b06d49cca284d89ab490b1bb6e69a01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255301135 2024-11-10T16:15:06,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742212_1388 (size=39749) 2024-11-10T16:15:06,518 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 598e521a0c4b4f6e99a3761ca695e15b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731255302315 2024-11-10T16:15:06,518 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 923e4c049e904f50a4f0716fd90f1744, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:06,526 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:06,526 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/9ac4a555c93244459b5254baa7af6400 is 50, key is test_row_0/C:col10/1731255305196/Put/seqid=0 2024-11-10T16:15:06,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742213_1389 (size=12561) 2024-11-10T16:15:06,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255366554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255366556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255366557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255366558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255366559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255366673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255366683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255366683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255366683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255366683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255366893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255366893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255366893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255366893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255366893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:06,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:06,916 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/1fa3516aa4d94882a4bfe0018f10e4fd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd 2024-11-10T16:15:06,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/11eb92d1ac7f4265b79c5030bea74e6a 2024-11-10T16:15:06,921 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/A of 307cbe8008d263744dfb710286b3c81a into 1fa3516aa4d94882a4bfe0018f10e4fd(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:06,921 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:06,921 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/A, priority=13, startTime=1731255306465; duration=0sec 2024-11-10T16:15:06,921 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:06,921 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:15:06,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/01aa14db87d94cbdb3f05a41a03cd769 is 50, key is test_row_0/B:col10/1731255305285/Put/seqid=0 2024-11-10T16:15:06,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742214_1390 (size=12151) 2024-11-10T16:15:06,938 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/9ac4a555c93244459b5254baa7af6400 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/9ac4a555c93244459b5254baa7af6400 2024-11-10T16:15:06,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:06,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:06,943 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/C of 307cbe8008d263744dfb710286b3c81a into 9ac4a555c93244459b5254baa7af6400(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:06,943 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:06,943 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/C, priority=13, startTime=1731255306465; duration=0sec 2024-11-10T16:15:06,943 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:06,943 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:15:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:07,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:07,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255367213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:07,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:07,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:07,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:07,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/01aa14db87d94cbdb3f05a41a03cd769 2024-11-10T16:15:07,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/8883b46580e1439f9d0e4f4b6561dc61 is 50, key is test_row_0/C:col10/1731255305285/Put/seqid=0 2024-11-10T16:15:07,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742215_1391 (size=12151) 2024-11-10T16:15:07,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/8883b46580e1439f9d0e4f4b6561dc61 2024-11-10T16:15:07,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/11eb92d1ac7f4265b79c5030bea74e6a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a 2024-11-10T16:15:07,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a, entries=200, sequenceid=174, filesize=38.8 K 2024-11-10T16:15:07,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/01aa14db87d94cbdb3f05a41a03cd769 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769 2024-11-10T16:15:07,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769, entries=150, sequenceid=174, filesize=11.9 K 2024-11-10T16:15:07,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/8883b46580e1439f9d0e4f4b6561dc61 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61 2024-11-10T16:15:07,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61, entries=150, sequenceid=174, filesize=11.9 K 2024-11-10T16:15:07,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 307cbe8008d263744dfb710286b3c81a in 887ms, sequenceid=174, compaction requested=false 2024-11-10T16:15:07,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:07,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T16:15:07,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:07,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:07,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d44f3379c1f241c0b8f896830912b657_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255306554/Put/seqid=0 2024-11-10T16:15:07,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742216_1392 (size=12304) 2024-11-10T16:15:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:07,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255367753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255367754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255367754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255367755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255367756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:07,819 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d44f3379c1f241c0b8f896830912b657_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d44f3379c1f241c0b8f896830912b657_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/3983a93be0894cc9b5c7c82f159bdae0, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:07,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/3983a93be0894cc9b5c7c82f159bdae0 is 175, key is test_row_0/A:col10/1731255306554/Put/seqid=0 2024-11-10T16:15:07,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742217_1393 (size=31105) 2024-11-10T16:15:07,829 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/3983a93be0894cc9b5c7c82f159bdae0 2024-11-10T16:15:07,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d149c3555de546ecb1c24c93b32a5f00 is 50, key is test_row_0/B:col10/1731255306554/Put/seqid=0 2024-11-10T16:15:07,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742218_1394 (size=12151) 2024-11-10T16:15:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255367874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255367874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255367874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255367874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:07,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255367874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:07,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255368093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255368093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255368093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255368093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255368093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,104 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,239 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d149c3555de546ecb1c24c93b32a5f00 2024-11-10T16:15:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/eaa7df6b722b4330952f6b2c8fbc263f is 50, key is test_row_0/C:col10/1731255306554/Put/seqid=0 2024-11-10T16:15:08,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742219_1395 (size=12151) 2024-11-10T16:15:08,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255368413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255368413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255368413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255368413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255368413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/eaa7df6b722b4330952f6b2c8fbc263f 2024-11-10T16:15:08,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/3983a93be0894cc9b5c7c82f159bdae0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0 2024-11-10T16:15:08,659 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0, entries=150, sequenceid=194, filesize=30.4 K 2024-11-10T16:15:08,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/d149c3555de546ecb1c24c93b32a5f00 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00 2024-11-10T16:15:08,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T16:15:08,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/eaa7df6b722b4330952f6b2c8fbc263f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f 2024-11-10T16:15:08,667 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f, entries=150, sequenceid=194, filesize=11.9 K 2024-11-10T16:15:08,668 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 307cbe8008d263744dfb710286b3c81a in 1267ms, sequenceid=194, compaction requested=true 2024-11-10T16:15:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:08,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T16:15:08,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T16:15:08,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T16:15:08,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8040 sec 2024-11-10T16:15:08,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.8090 sec 2024-11-10T16:15:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:08,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:08,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:08,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fc0585f8b644fb2be30db784a61c66d_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:08,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742220_1396 (size=14794) 2024-11-10T16:15:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255368959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255368961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255368962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255368967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255368971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:08,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255369083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255369083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255369083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255369083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255369083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255369293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255369293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255369293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255369303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255369303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,353 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:09,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107fc0585f8b644fb2be30db784a61c66d_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fc0585f8b644fb2be30db784a61c66d_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:09,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/7c9b6268d52e43b8b84ea51d8599309e, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:09,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/7c9b6268d52e43b8b84ea51d8599309e is 175, key is test_row_0/A:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:09,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742221_1397 (size=39749) 2024-11-10T16:15:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255369603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255369603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255369603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255369623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255369623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:09,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/7c9b6268d52e43b8b84ea51d8599309e 2024-11-10T16:15:09,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/342feb301f594316b6b7d0f9e847f6d0 is 50, key is test_row_0/B:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:09,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742222_1398 (size=12151) 2024-11-10T16:15:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T16:15:10,034 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:10,034 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-10T16:15:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:10,036 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:10,037 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:10,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:10,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255370123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255370123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255370123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:10,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:10,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255370144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255370144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:10,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:10,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/342feb301f594316b6b7d0f9e847f6d0 2024-11-10T16:15:10,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/83bce2c8c58945fa986ef178e76d8ed7 is 50, key is test_row_0/C:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:10,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742223_1399 (size=12151) 2024-11-10T16:15:10,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T16:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T16:15:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:10,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T16:15:10,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:10,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:10,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/83bce2c8c58945fa986ef178e76d8ed7 2024-11-10T16:15:10,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/7c9b6268d52e43b8b84ea51d8599309e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e 2024-11-10T16:15:10,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e, entries=200, sequenceid=210, filesize=38.8 K 2024-11-10T16:15:10,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/342feb301f594316b6b7d0f9e847f6d0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0 2024-11-10T16:15:10,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T16:15:10,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/83bce2c8c58945fa986ef178e76d8ed7 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7 2024-11-10T16:15:10,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T16:15:10,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 307cbe8008d263744dfb710286b3c81a in 1662ms, sequenceid=210, compaction requested=true 2024-11-10T16:15:10,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:10,603 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:10,603 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:10,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/A is initiating minor compaction (all files) 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/B is initiating minor compaction (all files) 2024-11-10T16:15:10,604 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/A in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,604 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/B in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,604 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=138.8 K 2024-11-10T16:15:10,604 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/634a4e73ebf34d85825f168d2fd0fbbb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=47.9 K 2024-11-10T16:15:10,604 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e] 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 634a4e73ebf34d85825f168d2fd0fbbb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:10,604 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa3516aa4d94882a4bfe0018f10e4fd, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 01aa14db87d94cbdb3f05a41a03cd769, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255305268 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11eb92d1ac7f4265b79c5030bea74e6a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255305257 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d149c3555de546ecb1c24c93b32a5f00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731255306554 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3983a93be0894cc9b5c7c82f159bdae0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731255306554 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 342feb301f594316b6b7d0f9e847f6d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731255307752 2024-11-10T16:15:10,605 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c9b6268d52e43b8b84ea51d8599309e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731255307752 2024-11-10T16:15:10,612 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:10,614 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411108354fe606d7c4749a2099e04dea2c9f7_307cbe8008d263744dfb710286b3c81a store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:10,615 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:10,615 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/e61a7dfcde9e4b7f849451ad0a45ae54 is 50, key is test_row_0/B:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:10,617 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411108354fe606d7c4749a2099e04dea2c9f7_307cbe8008d263744dfb710286b3c81a, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:10,617 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108354fe606d7c4749a2099e04dea2c9f7_307cbe8008d263744dfb710286b3c81a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:10,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742224_1400 (size=12697) 2024-11-10T16:15:10,637 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/e61a7dfcde9e4b7f849451ad0a45ae54 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e61a7dfcde9e4b7f849451ad0a45ae54 2024-11-10T16:15:10,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742225_1401 (size=4469) 2024-11-10T16:15:10,644 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#A#compaction#332 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:10,645 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/2d1cd014ae0b46a094355e48e85acdb3 is 175, key is test_row_0/A:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:10,645 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/B of 307cbe8008d263744dfb710286b3c81a into e61a7dfcde9e4b7f849451ad0a45ae54(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:10,645 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:10,645 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/B, priority=12, startTime=1731255310603; duration=0sec 2024-11-10T16:15:10,645 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:10,645 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:15:10,645 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:10,647 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:10,647 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 307cbe8008d263744dfb710286b3c81a/C is initiating minor compaction (all files) 2024-11-10T16:15:10,647 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 307cbe8008d263744dfb710286b3c81a/C in TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,648 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/9ac4a555c93244459b5254baa7af6400, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp, totalSize=47.9 K 2024-11-10T16:15:10,648 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac4a555c93244459b5254baa7af6400, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731255303950 2024-11-10T16:15:10,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-10T16:15:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:10,649 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8883b46580e1439f9d0e4f4b6561dc61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731255305268 2024-11-10T16:15:10,649 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T16:15:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742226_1402 (size=31651) 2024-11-10T16:15:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:10,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:10,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:10,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa7df6b722b4330952f6b2c8fbc263f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731255306554 2024-11-10T16:15:10,650 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 83bce2c8c58945fa986ef178e76d8ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731255307752 2024-11-10T16:15:10,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff219d210ca64ce2a0d973c0f7367b8c_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255308965/Put/seqid=0 2024-11-10T16:15:10,660 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 307cbe8008d263744dfb710286b3c81a#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:10,661 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/65ae6db916ed420493b1f0666007b83f is 50, key is test_row_0/C:col10/1731255307754/Put/seqid=0 2024-11-10T16:15:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:10,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742227_1403 (size=12304) 2024-11-10T16:15:10,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742228_1404 (size=12697) 2024-11-10T16:15:11,054 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/2d1cd014ae0b46a094355e48e85acdb3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/2d1cd014ae0b46a094355e48e85acdb3 2024-11-10T16:15:11,059 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/A of 307cbe8008d263744dfb710286b3c81a into 2d1cd014ae0b46a094355e48e85acdb3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:11,059 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:11,059 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/A, priority=12, startTime=1731255310602; duration=0sec 2024-11-10T16:15:11,060 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:11,060 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:15:11,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:11,066 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff219d210ca64ce2a0d973c0f7367b8c_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff219d210ca64ce2a0d973c0f7367b8c_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:11,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/05b165bfc60c483dacc257b142a336b9, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:11,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/05b165bfc60c483dacc257b142a336b9 is 175, key is test_row_0/A:col10/1731255308965/Put/seqid=0 2024-11-10T16:15:11,076 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/65ae6db916ed420493b1f0666007b83f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/65ae6db916ed420493b1f0666007b83f 2024-11-10T16:15:11,081 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 307cbe8008d263744dfb710286b3c81a/C of 307cbe8008d263744dfb710286b3c81a into 65ae6db916ed420493b1f0666007b83f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:11,081 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:11,081 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., storeName=307cbe8008d263744dfb710286b3c81a/C, priority=12, startTime=1731255310603; duration=0sec 2024-11-10T16:15:11,081 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:11,081 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:15:11,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742229_1405 (size=31105) 2024-11-10T16:15:11,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/05b165bfc60c483dacc257b142a336b9 2024-11-10T16:15:11,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/22176194802a4684a80c1f3a12f7cde2 is 50, key is test_row_0/B:col10/1731255308965/Put/seqid=0 2024-11-10T16:15:11,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742230_1406 (size=12151) 2024-11-10T16:15:11,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:11,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. as already flushing 2024-11-10T16:15:11,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:11,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255371172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255371174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255371175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255371176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255371187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,231 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-10T16:15:11,231 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:11,231 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,231 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-10T16:15:11,231 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,231 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:11,231 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:11,232 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:11,232 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-10T16:15:11,232 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:11,232 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,233 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:11,241 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-10T16:15:11,241 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:11,241 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,241 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:11,242 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-10T16:15:11,242 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:11,242 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,242 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:11,242 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255371293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255371294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255371293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255371294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255371303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,496 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/22176194802a4684a80c1f3a12f7cde2 2024-11-10T16:15:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/0a98230d22fe49f5a579c956a1f39399 is 50, key is test_row_0/C:col10/1731255308965/Put/seqid=0 2024-11-10T16:15:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255371503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255371503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255371503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255371504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742231_1407 (size=12151) 2024-11-10T16:15:11,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255371513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60506 deadline: 1731255371813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60514 deadline: 1731255371813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60440 deadline: 1731255371813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60426 deadline: 1731255371813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:11,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60460 deadline: 1731255371824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:11,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:11,906 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/0a98230d22fe49f5a579c956a1f39399 2024-11-10T16:15:11,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/05b165bfc60c483dacc257b142a336b9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/05b165bfc60c483dacc257b142a336b9 2024-11-10T16:15:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/05b165bfc60c483dacc257b142a336b9, entries=150, sequenceid=227, filesize=30.4 K 2024-11-10T16:15:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/22176194802a4684a80c1f3a12f7cde2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/22176194802a4684a80c1f3a12f7cde2 2024-11-10T16:15:11,916 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/22176194802a4684a80c1f3a12f7cde2, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T16:15:11,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/0a98230d22fe49f5a579c956a1f39399 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/0a98230d22fe49f5a579c956a1f39399 2024-11-10T16:15:11,919 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/0a98230d22fe49f5a579c956a1f39399, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T16:15:11,920 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 307cbe8008d263744dfb710286b3c81a in 1270ms, sequenceid=227, compaction requested=false 2024-11-10T16:15:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:11,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-10T16:15:11,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-10T16:15:11,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T16:15:11,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-10T16:15:11,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8890 sec 2024-11-10T16:15:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T16:15:12,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:12,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:12,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:15:12,325 INFO [Thread-1529 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1529. 2024-11-10T16:15:12,325 INFO [Thread-1531 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1531. 2024-11-10T16:15:12,325 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-10T16:15:12,325 DEBUG [Thread-1529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:12,325 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:12,325 DEBUG [Thread-1531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:12,325 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 DEBUG [Thread-1529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 DEBUG [Thread-1531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:12,326 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-10T16:15:12,326 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:12,326 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:12,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:12,326 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c01da737384e431dbd28b1fb69fe083a_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_0/A:col10/1731255312323/Put/seqid=0 2024-11-10T16:15:12,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742232_1408 (size=12304) 2024-11-10T16:15:12,335 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-10T16:15:12,335 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:12,335 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,335 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3254 rows 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1086 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3257 rows 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3248 rows 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3249 rows 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-10T16:15:12,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3245 rows 2024-11-10T16:15:12,335 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:15:12,335 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:15:12,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:12,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:12,336 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:15:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:12,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:12,339 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255312339"}]},"ts":"1731255312339"} 2024-11-10T16:15:12,340 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:15:12,340 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:15:12,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:15:12,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, UNASSIGN}] 2024-11-10T16:15:12,342 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, UNASSIGN 2024-11-10T16:15:12,343 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:12,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, UNASSIGN because future has completed 2024-11-10T16:15:12,344 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:15:12,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:12,496 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:12,496 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:15:12,496 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 307cbe8008d263744dfb710286b3c81a, disabling compactions & flushes 2024-11-10T16:15:12,496 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:12,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:12,739 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c01da737384e431dbd28b1fb69fe083a_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c01da737384e431dbd28b1fb69fe083a_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:12,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/e2fd1db9d6da4de585f5696a9b7ffcd3, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:12,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/e2fd1db9d6da4de585f5696a9b7ffcd3 is 175, key is test_row_0/A:col10/1731255312323/Put/seqid=0 2024-11-10T16:15:12,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742233_1409 (size=31105) 2024-11-10T16:15:12,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:13,144 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/e2fd1db9d6da4de585f5696a9b7ffcd3 2024-11-10T16:15:13,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/7f555998544b4e80ba634f7c4767e0a1 is 50, key is test_row_0/B:col10/1731255312323/Put/seqid=0 2024-11-10T16:15:13,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742234_1410 (size=12151) 2024-11-10T16:15:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:13,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/7f555998544b4e80ba634f7c4767e0a1 2024-11-10T16:15:13,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/19bc68ec5bbe486a90fa19c3c65e6342 is 50, key is test_row_0/C:col10/1731255312323/Put/seqid=0 2024-11-10T16:15:13,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742235_1411 (size=12151) 2024-11-10T16:15:13,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/19bc68ec5bbe486a90fa19c3c65e6342 2024-11-10T16:15:13,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/e2fd1db9d6da4de585f5696a9b7ffcd3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/e2fd1db9d6da4de585f5696a9b7ffcd3 2024-11-10T16:15:13,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/e2fd1db9d6da4de585f5696a9b7ffcd3, entries=150, sequenceid=249, filesize=30.4 K 2024-11-10T16:15:13,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/7f555998544b4e80ba634f7c4767e0a1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/7f555998544b4e80ba634f7c4767e0a1 2024-11-10T16:15:13,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/7f555998544b4e80ba634f7c4767e0a1, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T16:15:13,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/19bc68ec5bbe486a90fa19c3c65e6342 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/19bc68ec5bbe486a90fa19c3c65e6342 2024-11-10T16:15:13,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/19bc68ec5bbe486a90fa19c3c65e6342, entries=150, sequenceid=249, filesize=11.9 K 2024-11-10T16:15:13,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=6.71 KB/6870 for 307cbe8008d263744dfb710286b3c81a in 1651ms, sequenceid=249, compaction requested=true 2024-11-10T16:15:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 307cbe8008d263744dfb710286b3c81a: 2024-11-10T16:15:13,976 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:13,976 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. because compaction request was cancelled 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. after waiting 1 ms 2024-11-10T16:15:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:A 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. because compaction request was cancelled 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:B 2024-11-10T16:15:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 307cbe8008d263744dfb710286b3c81a:C, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. because compaction request was cancelled 2024-11-10T16:15:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:13,977 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 307cbe8008d263744dfb710286b3c81a:C 2024-11-10T16:15:13,977 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 307cbe8008d263744dfb710286b3c81a 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=A 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=B 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 307cbe8008d263744dfb710286b3c81a, store=C 2024-11-10T16:15:13,977 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:13,981 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111098da3a78f5b54ecea919b996d8db1918_307cbe8008d263744dfb710286b3c81a is 50, key is test_row_2/A:col10/1731255312334/Put/seqid=0 2024-11-10T16:15:13,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742236_1412 (size=7324) 2024-11-10T16:15:14,385 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:14,388 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111098da3a78f5b54ecea919b996d8db1918_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111098da3a78f5b54ecea919b996d8db1918_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:14,389 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41d45a91ae874ad8a4973dba84a59d65, store: [table=TestAcidGuarantees family=A region=307cbe8008d263744dfb710286b3c81a] 2024-11-10T16:15:14,389 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41d45a91ae874ad8a4973dba84a59d65 is 175, key is test_row_2/A:col10/1731255312334/Put/seqid=0 2024-11-10T16:15:14,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742237_1413 (size=13815) 2024-11-10T16:15:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:14,793 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41d45a91ae874ad8a4973dba84a59d65 2024-11-10T16:15:14,799 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/05f18f608eba4d84a05f239f721b1f9e is 50, key is test_row_2/B:col10/1731255312334/Put/seqid=0 2024-11-10T16:15:14,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742238_1414 (size=7365) 2024-11-10T16:15:15,202 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/05f18f608eba4d84a05f239f721b1f9e 2024-11-10T16:15:15,208 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/e2af07954ce64b5ba04ce980258b7f5f is 50, key is test_row_2/C:col10/1731255312334/Put/seqid=0 2024-11-10T16:15:15,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742239_1415 (size=7365) 2024-11-10T16:15:15,611 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/e2af07954ce64b5ba04ce980258b7f5f 2024-11-10T16:15:15,615 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/A/41d45a91ae874ad8a4973dba84a59d65 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41d45a91ae874ad8a4973dba84a59d65 2024-11-10T16:15:15,618 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41d45a91ae874ad8a4973dba84a59d65, entries=50, sequenceid=253, filesize=13.5 K 2024-11-10T16:15:15,618 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/B/05f18f608eba4d84a05f239f721b1f9e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/05f18f608eba4d84a05f239f721b1f9e 2024-11-10T16:15:15,621 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/05f18f608eba4d84a05f239f721b1f9e, entries=50, sequenceid=253, filesize=7.2 K 2024-11-10T16:15:15,622 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/.tmp/C/e2af07954ce64b5ba04ce980258b7f5f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/e2af07954ce64b5ba04ce980258b7f5f 2024-11-10T16:15:15,624 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/e2af07954ce64b5ba04ce980258b7f5f, entries=50, sequenceid=253, filesize=7.2 K 2024-11-10T16:15:15,625 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 307cbe8008d263744dfb710286b3c81a in 1648ms, sequenceid=253, compaction requested=true 2024-11-10T16:15:15,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e] to archive 2024-11-10T16:15:15,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:15,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/96f00c3067ad48ebb61c22e3333d3336 2024-11-10T16:15:15,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41972d6d0333455f911c98604fd2a7d7 2024-11-10T16:15:15,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/d1df3be97f6d4b5cbc26b1fdf12f0fa9 2024-11-10T16:15:15,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/20563eda1b6f4fa882ba726f2c4c740e 2024-11-10T16:15:15,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a00143ff8f7b4da8a10a564537ab70c2 2024-11-10T16:15:15,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/570a90f530ad450496b5746341b171c9 2024-11-10T16:15:15,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/22b26b0dc0104d76a0e528863b706ad2 2024-11-10T16:15:15,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/6d5e118d705f4aeb976b50deabfc98b3 2024-11-10T16:15:15,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/a7aaf96deca74655987c9b2ccfc25027 2024-11-10T16:15:15,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/8cd8aae977d2420190478d612ac11bcc 2024-11-10T16:15:15,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/fd1caa5f825240ccaf0e74ea4e4e7b87 2024-11-10T16:15:15,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/313901096ef44d6fb97a72353f0b1187 2024-11-10T16:15:15,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/1fa3516aa4d94882a4bfe0018f10e4fd 2024-11-10T16:15:15,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/11eb92d1ac7f4265b79c5030bea74e6a 2024-11-10T16:15:15,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/3983a93be0894cc9b5c7c82f159bdae0 2024-11-10T16:15:15,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/7c9b6268d52e43b8b84ea51d8599309e 2024-11-10T16:15:15,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/b8209d982fe2463e9a1bb65b28f26528, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/a7d9d51f1f21404f839c8ac51fe1876c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/257c0b941a2e4b5b969251b50f3a622d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/634a4e73ebf34d85825f168d2fd0fbbb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0] to archive 2024-11-10T16:15:15,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:15,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d1b8bce9a4924f6194369114e125df80 2024-11-10T16:15:15,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/258f96a862414e439fc7368706ce90f9 2024-11-10T16:15:15,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/b8209d982fe2463e9a1bb65b28f26528 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/b8209d982fe2463e9a1bb65b28f26528 2024-11-10T16:15:15,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3426c592860c4d7dbfdb1e359087376c 2024-11-10T16:15:15,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/751a4c1de7254b9eb0731fbc550ffeba 2024-11-10T16:15:15,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/a7d9d51f1f21404f839c8ac51fe1876c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/a7d9d51f1f21404f839c8ac51fe1876c 2024-11-10T16:15:15,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/bae14bc4e3f74139b263787fc1196fb8 2024-11-10T16:15:15,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e8eb03125a94466e97e5a37868466059 2024-11-10T16:15:15,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/257c0b941a2e4b5b969251b50f3a622d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/257c0b941a2e4b5b969251b50f3a622d 2024-11-10T16:15:15,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/51a55332752d4bd3a4c8b726c615cfa3 2024-11-10T16:15:15,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/3af137dc0c8e435487ef33ec41227a08 2024-11-10T16:15:15,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/634a4e73ebf34d85825f168d2fd0fbbb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/634a4e73ebf34d85825f168d2fd0fbbb 2024-11-10T16:15:15,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/4d090cde54844432bb04c1d59841045c 2024-11-10T16:15:15,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/01aa14db87d94cbdb3f05a41a03cd769 2024-11-10T16:15:15,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/d149c3555de546ecb1c24c93b32a5f00 2024-11-10T16:15:15,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/342feb301f594316b6b7d0f9e847f6d0 2024-11-10T16:15:15,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/4b15e48ae9b84d63ae4d896ea6dcb4e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/a8dbf97fc85d4630b5e1d6a5a90d2122, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/5b06d49cca284d89ab490b1bb6e69a01, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/9ac4a555c93244459b5254baa7af6400, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7] to archive 2024-11-10T16:15:15,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:15,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/3180e5b012d547b8b116cd82117e8038 2024-11-10T16:15:15,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/1f3a478019ac4814a0f8e347fe9ee0db 2024-11-10T16:15:15,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/4b15e48ae9b84d63ae4d896ea6dcb4e3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/4b15e48ae9b84d63ae4d896ea6dcb4e3 2024-11-10T16:15:15,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/647cbc6e84f9494d99b2311431089cfc 2024-11-10T16:15:15,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/d7e501be792644a4b7701d3fd9f506b9 2024-11-10T16:15:15,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/a8dbf97fc85d4630b5e1d6a5a90d2122 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/a8dbf97fc85d4630b5e1d6a5a90d2122 2024-11-10T16:15:15,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/55068b97a974442eb21fe7af11688f83 2024-11-10T16:15:15,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/7b8f402544454deea5f9c2b540210fc9 2024-11-10T16:15:15,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/5b06d49cca284d89ab490b1bb6e69a01 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/5b06d49cca284d89ab490b1bb6e69a01 2024-11-10T16:15:15,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/08716ad9483c40efaaf4c620579b610f 2024-11-10T16:15:15,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/598e521a0c4b4f6e99a3761ca695e15b 2024-11-10T16:15:15,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/9ac4a555c93244459b5254baa7af6400 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/9ac4a555c93244459b5254baa7af6400 2024-11-10T16:15:15,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/923e4c049e904f50a4f0716fd90f1744 2024-11-10T16:15:15,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/8883b46580e1439f9d0e4f4b6561dc61 2024-11-10T16:15:15,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/eaa7df6b722b4330952f6b2c8fbc263f 2024-11-10T16:15:15,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/83bce2c8c58945fa986ef178e76d8ed7 2024-11-10T16:15:15,673 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits/256.seqid, newMaxSeqId=256, maxSeqId=4 2024-11-10T16:15:15,673 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. 2024-11-10T16:15:15,673 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 307cbe8008d263744dfb710286b3c81a: Waiting for close lock at 1731255312496Running coprocessor pre-close hooks at 1731255312496Disabling compacts and flushes for region at 1731255312496Disabling writes for close at 1731255313977 (+1481 ms)Obtaining lock to block concurrent updates at 1731255313977Preparing flush snapshotting stores in 307cbe8008d263744dfb710286b3c81a at 1731255313977Finished memstore snapshotting TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731255313977Flushing stores of TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a. at 1731255313977Flushing 307cbe8008d263744dfb710286b3c81a/A: creating writer at 1731255313977Flushing mob file 307cbe8008d263744dfb710286b3c81a/A: appending metadata at 1731255313981 (+4 ms)Flushing mob file 307cbe8008d263744dfb710286b3c81a/A: closing flushed file at 1731255313981Flushing 307cbe8008d263744dfb710286b3c81a/A: appending metadata at 1731255314389 (+408 ms)Flushing 307cbe8008d263744dfb710286b3c81a/A: closing flushed file at 1731255314389Flushing 307cbe8008d263744dfb710286b3c81a/B: creating writer at 1731255314796 (+407 ms)Flushing 307cbe8008d263744dfb710286b3c81a/B: appending metadata at 1731255314798 (+2 ms)Flushing 307cbe8008d263744dfb710286b3c81a/B: closing flushed file at 1731255314798Flushing 307cbe8008d263744dfb710286b3c81a/C: creating writer at 1731255315205 (+407 ms)Flushing 307cbe8008d263744dfb710286b3c81a/C: appending metadata at 1731255315207 (+2 ms)Flushing 307cbe8008d263744dfb710286b3c81a/C: closing flushed file at 1731255315207Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c3a5fa: reopening flushed file at 1731255315615 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b8f89d2: reopening flushed file at 1731255315618 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19a7d01: reopening flushed file at 1731255315621 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for 307cbe8008d263744dfb710286b3c81a in 1648ms, sequenceid=253, compaction requested=true at 1731255315625 (+4 ms)Writing region close event to WAL at 1731255315670 (+45 ms)Running coprocessor post-close hooks at 1731255315673 (+3 ms)Closed at 1731255315673 2024-11-10T16:15:15,675 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:15,676 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=307cbe8008d263744dfb710286b3c81a, regionState=CLOSED 2024-11-10T16:15:15,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:15,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-10T16:15:15,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 307cbe8008d263744dfb710286b3c81a, server=8e3de9160f9f,44105,1731255200149 in 3.3330 sec 2024-11-10T16:15:15,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-10T16:15:15,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=307cbe8008d263744dfb710286b3c81a, UNASSIGN in 3.3360 sec 2024-11-10T16:15:15,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T16:15:15,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3390 sec 2024-11-10T16:15:15,682 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255315682"}]},"ts":"1731255315682"} 2024-11-10T16:15:15,683 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:15:15,683 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:15:15,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3470 sec 2024-11-10T16:15:16,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-10T16:15:16,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:16,504 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:15:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T16:15:16,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,509 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,510 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits] 2024-11-10T16:15:16,512 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/05b165bfc60c483dacc257b142a336b9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/05b165bfc60c483dacc257b142a336b9 2024-11-10T16:15:16,513 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/2d1cd014ae0b46a094355e48e85acdb3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/2d1cd014ae0b46a094355e48e85acdb3 2024-11-10T16:15:16,514 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41d45a91ae874ad8a4973dba84a59d65 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/41d45a91ae874ad8a4973dba84a59d65 2024-11-10T16:15:16,515 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/e2fd1db9d6da4de585f5696a9b7ffcd3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/A/e2fd1db9d6da4de585f5696a9b7ffcd3 2024-11-10T16:15:16,517 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/05f18f608eba4d84a05f239f721b1f9e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/05f18f608eba4d84a05f239f721b1f9e 2024-11-10T16:15:16,518 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/22176194802a4684a80c1f3a12f7cde2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/22176194802a4684a80c1f3a12f7cde2 2024-11-10T16:15:16,518 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/7f555998544b4e80ba634f7c4767e0a1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/7f555998544b4e80ba634f7c4767e0a1 2024-11-10T16:15:16,519 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e61a7dfcde9e4b7f849451ad0a45ae54 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/B/e61a7dfcde9e4b7f849451ad0a45ae54 2024-11-10T16:15:16,521 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/0a98230d22fe49f5a579c956a1f39399 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/0a98230d22fe49f5a579c956a1f39399 2024-11-10T16:15:16,522 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/19bc68ec5bbe486a90fa19c3c65e6342 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/19bc68ec5bbe486a90fa19c3c65e6342 2024-11-10T16:15:16,523 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/65ae6db916ed420493b1f0666007b83f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/65ae6db916ed420493b1f0666007b83f 2024-11-10T16:15:16,523 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/e2af07954ce64b5ba04ce980258b7f5f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/C/e2af07954ce64b5ba04ce980258b7f5f 2024-11-10T16:15:16,525 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits/256.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a/recovered.edits/256.seqid 2024-11-10T16:15:16,526 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,526 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:15:16,526 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:15:16,527 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T16:15:16,529 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111008381ac802e046ce8807cc1f782f3b1d_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111008381ac802e046ce8807cc1f782f3b1d_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014f3307e361c4d36b9bc160531f369fa_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014f3307e361c4d36b9bc160531f369fa_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,531 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026766b27809a4ed3a80c096e95fa4788_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111026766b27809a4ed3a80c096e95fa4788_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,531 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110279d0136fac2454e9e4d50558c47d4a8_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110279d0136fac2454e9e4d50558c47d4a8_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,532 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103733e126cbdd448ba5ca28ff8ca5e4d4_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103733e126cbdd448ba5ca28ff8ca5e4d4_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103819b7e2534840b59f9ad9127744bff9_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103819b7e2534840b59f9ad9127744bff9_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,534 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110464f44f9f06d4804ae997b8e8b846019_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110464f44f9f06d4804ae997b8e8b846019_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,535 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fc0585f8b644fb2be30db784a61c66d_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107fc0585f8b644fb2be30db784a61c66d_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,536 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109062b001254c4720a346978fa91740d3_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109062b001254c4720a346978fa91740d3_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,537 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109556038b7ec244d2ae3a0b8e16a3a44e_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109556038b7ec244d2ae3a0b8e16a3a44e_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,538 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111098da3a78f5b54ecea919b996d8db1918_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111098da3a78f5b54ecea919b996d8db1918_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,538 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c01da737384e431dbd28b1fb69fe083a_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110c01da737384e431dbd28b1fb69fe083a_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,539 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d44f3379c1f241c0b8f896830912b657_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d44f3379c1f241c0b8f896830912b657_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,540 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fa3108d68088489ba7daaea5e0720c46_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110fa3108d68088489ba7daaea5e0720c46_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,542 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff219d210ca64ce2a0d973c0f7367b8c_307cbe8008d263744dfb710286b3c81a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff219d210ca64ce2a0d973c0f7367b8c_307cbe8008d263744dfb710286b3c81a 2024-11-10T16:15:16,542 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:15:16,544 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,546 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:15:16,547 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:15:16,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,548 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:15:16,548 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255316548"}]},"ts":"9223372036854775807"} 2024-11-10T16:15:16,550 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:15:16,550 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 307cbe8008d263744dfb710286b3c81a, NAME => 'TestAcidGuarantees,,1731255288113.307cbe8008d263744dfb710286b3c81a.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:15:16,550 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:15:16,550 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255316550"}]},"ts":"9223372036854775807"} 2024-11-10T16:15:16,551 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:15:16,552 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-10T16:15:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-10T16:15:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:15:16,613 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:16,622 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250 (was 252), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 342), ProcessCount=11 (was 11), AvailableMemoryMB=7039 (was 7060) 2024-11-10T16:15:16,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=7039 2024-11-10T16:15:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:15:16,632 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:15:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:16,634 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:15:16,634 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:16,634 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-10T16:15:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:16,635 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:15:16,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742240_1416 (size=1061) 2024-11-10T16:15:16,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:17,041 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:15:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742241_1417 (size=53) 2024-11-10T16:15:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 76c86cef11e7fc54de9404956c98cfb7, disabling compactions & flushes 2024-11-10T16:15:17,447 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. after waiting 0 ms 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,447 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,447 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 76c86cef11e7fc54de9404956c98cfb7: Waiting for close lock at 1731255317447Disabling compacts and flushes for region at 1731255317447Disabling writes for close at 1731255317447Writing region close event to WAL at 1731255317447Closed at 1731255317447 2024-11-10T16:15:17,448 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:15:17,448 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255317448"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255317448"}]},"ts":"1731255317448"} 2024-11-10T16:15:17,450 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:15:17,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:15:17,450 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255317450"}]},"ts":"1731255317450"} 2024-11-10T16:15:17,451 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:15:17,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, ASSIGN}] 2024-11-10T16:15:17,453 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, ASSIGN 2024-11-10T16:15:17,453 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:15:17,604 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=76c86cef11e7fc54de9404956c98cfb7, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:17,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, ASSIGN because future has completed 2024-11-10T16:15:17,606 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:17,760 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,760 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:15:17,761 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,761 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:15:17,762 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,762 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,763 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,764 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:17,764 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c86cef11e7fc54de9404956c98cfb7 columnFamilyName A 2024-11-10T16:15:17,764 DEBUG [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:17,764 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(327): Store=76c86cef11e7fc54de9404956c98cfb7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:17,764 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,765 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:17,765 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c86cef11e7fc54de9404956c98cfb7 columnFamilyName B 2024-11-10T16:15:17,765 DEBUG [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:17,766 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(327): Store=76c86cef11e7fc54de9404956c98cfb7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:17,766 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,766 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:17,767 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 76c86cef11e7fc54de9404956c98cfb7 columnFamilyName C 2024-11-10T16:15:17,767 DEBUG [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:17,767 INFO [StoreOpener-76c86cef11e7fc54de9404956c98cfb7-1 {}] regionserver.HStore(327): Store=76c86cef11e7fc54de9404956c98cfb7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:17,767 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,767 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,768 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,768 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,768 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,769 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,769 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:15:17,770 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,771 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:15:17,771 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 76c86cef11e7fc54de9404956c98cfb7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65551194, jitterRate=-0.02321109175682068}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:15:17,771 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:17,772 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 76c86cef11e7fc54de9404956c98cfb7: Running coprocessor pre-open hook at 1731255317762Writing region info on filesystem at 1731255317762Initializing all the Stores at 1731255317762Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255317762Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255317762Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255317762Cleaning up temporary data from old regions at 1731255317769 (+7 ms)Running coprocessor post-open hooks at 1731255317771 (+2 ms)Region opened successfully at 1731255317772 (+1 ms) 2024-11-10T16:15:17,773 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., pid=116, masterSystemTime=1731255317757 2024-11-10T16:15:17,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:17,774 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,774 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:17,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=76c86cef11e7fc54de9404956c98cfb7, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:17,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:17,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T16:15:17,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 in 170 msec 2024-11-10T16:15:17,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-10T16:15:17,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, ASSIGN in 325 msec 2024-11-10T16:15:17,779 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:15:17,779 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255317779"}]},"ts":"1731255317779"} 2024-11-10T16:15:17,780 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:15:17,781 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:15:17,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-10T16:15:18,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T16:15:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-10T16:15:18,783 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:18,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,785 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bcd3a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,787 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34252db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,788 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,789 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,790 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f969e3a 2024-11-10T16:15:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:15:18,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:15:18,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44a445c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4e7859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21158d2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,797 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f40a1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c27f1e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,799 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484461aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,801 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28b48750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,802 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663bcfc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff7674f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,803 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d643714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,805 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d51d821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2c7469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b67c7db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cfc665d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,809 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645c1b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5919153d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b2dca8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,813 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,813 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,813 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0e8c2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,815 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52856, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6e36dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4df5a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:18,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:18,816 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1513119c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:18,817 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29be1f89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:18,818 DEBUG [Thread-1897 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,818 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,818 DEBUG [Thread-1890 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,818 DEBUG [Thread-1892 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,819 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,819 DEBUG [Thread-1899 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,819 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,820 DEBUG [Thread-1884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,821 DEBUG [Thread-1901 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,821 DEBUG [Thread-1901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,821 DEBUG [Thread-1888 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,822 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:18,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,822 DEBUG [Thread-1895 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:18,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:18,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,824 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T16:15:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:18,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,826 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:18,826 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,827 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:18,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2] 2024-11-10T16:15:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:18,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:15:18,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:18,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:18,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f31d0dced22947dcab547e82974d3557 is 50, key is test_row_0/A:col10/1731255318837/Put/seqid=0 2024-11-10T16:15:18,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742242_1418 (size=12001) 2024-11-10T16:15:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255378865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255378867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255378873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255378874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255378875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:18,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:18,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:18,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255378984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255378993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255378993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255379193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255379193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255379193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255379213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255379213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f31d0dced22947dcab547e82974d3557 2024-11-10T16:15:19,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/cd0bc2f391bc4e6cabd19b269ce94ad0 is 50, key is test_row_0/B:col10/1731255318837/Put/seqid=0 2024-11-10T16:15:19,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742243_1419 (size=12001) 2024-11-10T16:15:19,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255379513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255379513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255379513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255379533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255379534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:19,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/cd0bc2f391bc4e6cabd19b269ce94ad0 2024-11-10T16:15:19,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/567acbdfe0a5450da1fd09ceaf6a65b3 is 50, key is test_row_0/C:col10/1731255318837/Put/seqid=0 2024-11-10T16:15:19,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742244_1420 (size=12001) 2024-11-10T16:15:19,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:19,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:19,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255380033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:20,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255380033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255380034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:20,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255380043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:20,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255380043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:20,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:20,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:20,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:20,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:20,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:20,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/567acbdfe0a5450da1fd09ceaf6a65b3 2024-11-10T16:15:20,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f31d0dced22947dcab547e82974d3557 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557 2024-11-10T16:15:20,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T16:15:20,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/cd0bc2f391bc4e6cabd19b269ce94ad0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0 2024-11-10T16:15:20,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T16:15:20,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/567acbdfe0a5450da1fd09ceaf6a65b3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3 2024-11-10T16:15:20,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-10T16:15:20,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 76c86cef11e7fc54de9404956c98cfb7 in 1299ms, sequenceid=14, compaction requested=false 2024-11-10T16:15:20,138 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T16:15:20,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:20,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T16:15:20,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:20,205 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:20,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:20,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/594e36b86f7947a69111829153a16306 is 50, key is test_row_0/A:col10/1731255318865/Put/seqid=0 2024-11-10T16:15:20,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742245_1421 (size=12001) 2024-11-10T16:15:20,215 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/594e36b86f7947a69111829153a16306 2024-11-10T16:15:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d040610836a640009ffc62f99b73fb09 is 50, key is test_row_0/B:col10/1731255318865/Put/seqid=0 2024-11-10T16:15:20,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742246_1422 (size=12001) 2024-11-10T16:15:20,631 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d040610836a640009ffc62f99b73fb09 2024-11-10T16:15:20,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/5b358c1c31c44413bcc0b1c389382348 is 50, key is test_row_0/C:col10/1731255318865/Put/seqid=0 2024-11-10T16:15:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742247_1423 (size=12001) 2024-11-10T16:15:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:21,044 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/5b358c1c31c44413bcc0b1c389382348 2024-11-10T16:15:21,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/594e36b86f7947a69111829153a16306 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306 2024-11-10T16:15:21,051 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T16:15:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d040610836a640009ffc62f99b73fb09 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09 2024-11-10T16:15:21,056 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T16:15:21,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:21,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/5b358c1c31c44413bcc0b1c389382348 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348 2024-11-10T16:15:21,061 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348, entries=150, sequenceid=33, filesize=11.7 K 2024-11-10T16:15:21,062 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=13.42 KB/13740 for 76c86cef11e7fc54de9404956c98cfb7 in 857ms, sequenceid=33, compaction requested=false 2024-11-10T16:15:21,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:21,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:21,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T16:15:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T16:15:21,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T16:15:21,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-10T16:15:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:21,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:21,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:21,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:21,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:21,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:21,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2450 sec 2024-11-10T16:15:21,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/d90b69df58d7411d8a827bb20be7947a is 50, key is test_row_0/A:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:21,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742248_1424 (size=16681) 2024-11-10T16:15:21,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/d90b69df58d7411d8a827bb20be7947a 2024-11-10T16:15:21,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5077aced6d4944e19555c630a9bfeb07 is 50, key is test_row_0/B:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:21,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742249_1425 (size=12001) 2024-11-10T16:15:21,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255381103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255381113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255381114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255381114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255381115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,194 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:15:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255381224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255381224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255381233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255381233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255381233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255381443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255381443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255381453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255381453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255381453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5077aced6d4944e19555c630a9bfeb07 2024-11-10T16:15:21,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b0869b7ecc7642db8e59fc3366526165 is 50, key is test_row_0/C:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:21,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742250_1426 (size=12001) 2024-11-10T16:15:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255381753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255381754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255381773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255381773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,784 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255381773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:21,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:21,906 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b0869b7ecc7642db8e59fc3366526165 2024-11-10T16:15:21,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/d90b69df58d7411d8a827bb20be7947a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a 2024-11-10T16:15:21,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a, entries=250, sequenceid=45, filesize=16.3 K 2024-11-10T16:15:21,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5077aced6d4944e19555c630a9bfeb07 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07 2024-11-10T16:15:21,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07, entries=150, sequenceid=45, filesize=11.7 K 2024-11-10T16:15:21,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b0869b7ecc7642db8e59fc3366526165 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165 2024-11-10T16:15:21,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165, entries=150, sequenceid=45, filesize=11.7 K 2024-11-10T16:15:21,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76c86cef11e7fc54de9404956c98cfb7 in 857ms, sequenceid=45, compaction requested=true 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:21,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:21,924 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:21,924 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:21,925 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:21,925 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:21,925 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:21,925 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:21,925 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:21,925 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:21,925 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=35.2 K 2024-11-10T16:15:21,925 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=39.7 K 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting cd0bc2f391bc4e6cabd19b269ce94ad0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255318837 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f31d0dced22947dcab547e82974d3557, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255318837 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d040610836a640009ffc62f99b73fb09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731255318865 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 594e36b86f7947a69111829153a16306, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731255318865 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5077aced6d4944e19555c630a9bfeb07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:21,926 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90b69df58d7411d8a827bb20be7947a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:21,933 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:21,934 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:21,934 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/8e966095930947d6a980e519a5484494 is 50, key is test_row_0/A:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:21,934 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/2ce3aa89e71944f29b15c06603c51239 is 50, key is test_row_0/B:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742251_1427 (size=12104) 2024-11-10T16:15:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742252_1428 (size=12104) 2024-11-10T16:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:22,280 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:22,284 WARN [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-10T16:15:22,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e6e8a7d2eee648289f0a728d8ddb7460 is 50, key is test_row_0/A:col10/1731255321112/Put/seqid=0 2024-11-10T16:15:22,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742253_1429 (size=14341) 2024-11-10T16:15:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255382295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255382305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255382306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255382307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255382307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,347 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/2ce3aa89e71944f29b15c06603c51239 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/2ce3aa89e71944f29b15c06603c51239 2024-11-10T16:15:22,347 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/8e966095930947d6a980e519a5484494 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/8e966095930947d6a980e519a5484494 2024-11-10T16:15:22,351 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into 2ce3aa89e71944f29b15c06603c51239(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:22,351 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into 8e966095930947d6a980e519a5484494(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:22,351 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=13, startTime=1731255321924; duration=0sec 2024-11-10T16:15:22,351 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=13, startTime=1731255321924; duration=0sec 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:22,351 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:22,352 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:22,352 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:22,352 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:22,352 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=35.2 K 2024-11-10T16:15:22,353 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567acbdfe0a5450da1fd09ceaf6a65b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731255318837 2024-11-10T16:15:22,353 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b358c1c31c44413bcc0b1c389382348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731255318865 2024-11-10T16:15:22,353 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0869b7ecc7642db8e59fc3366526165, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:22,360 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:22,360 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/327110a6d0584630b1efb2eff668723e is 50, key is test_row_0/C:col10/1731255321067/Put/seqid=0 2024-11-10T16:15:22,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742254_1430 (size=12104) 2024-11-10T16:15:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255382414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255382414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255382414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255382433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255382433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255382633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255382633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255382633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255382643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255382643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e6e8a7d2eee648289f0a728d8ddb7460 2024-11-10T16:15:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c655505c8b244c64b0be907f0ec549ef is 50, key is test_row_0/B:col10/1731255321112/Put/seqid=0 2024-11-10T16:15:22,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742255_1431 (size=12001) 2024-11-10T16:15:22,768 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/327110a6d0584630b1efb2eff668723e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/327110a6d0584630b1efb2eff668723e 2024-11-10T16:15:22,772 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into 327110a6d0584630b1efb2eff668723e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:22,772 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:22,772 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=13, startTime=1731255321924; duration=0sec 2024-11-10T16:15:22,772 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:22,772 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255382953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255382953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255382953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255382963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255382963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T16:15:22,993 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:22,993 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T16:15:22,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:22,995 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:22,996 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:22,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:23,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c655505c8b244c64b0be907f0ec549ef 2024-11-10T16:15:23,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:23,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/fce05845483e465785bd62ffb26cb826 is 50, key is test_row_0/C:col10/1731255321112/Put/seqid=0 2024-11-10T16:15:23,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742256_1432 (size=12001) 2024-11-10T16:15:23,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T16:15:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T16:15:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:23,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T16:15:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255383474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:23,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:23,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255383474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:23,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:23,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255383474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255383483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:23,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:23,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255383483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:23,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:23,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/fce05845483e465785bd62ffb26cb826 2024-11-10T16:15:23,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e6e8a7d2eee648289f0a728d8ddb7460 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460 2024-11-10T16:15:23,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460, entries=200, sequenceid=66, filesize=14.0 K 2024-11-10T16:15:23,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c655505c8b244c64b0be907f0ec549ef as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef 2024-11-10T16:15:23,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef, entries=150, sequenceid=66, filesize=11.7 K 2024-11-10T16:15:23,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/fce05845483e465785bd62ffb26cb826 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826 2024-11-10T16:15:23,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826, entries=150, sequenceid=66, filesize=11.7 K 2024-11-10T16:15:23,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 76c86cef11e7fc54de9404956c98cfb7 in 1249ms, sequenceid=66, compaction requested=false 2024-11-10T16:15:23,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:23,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:23,608 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:23,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:23,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:23,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e110c06b56f8496eb39857784b2417f1 is 50, key is test_row_0/A:col10/1731255322294/Put/seqid=0 2024-11-10T16:15:23,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742257_1433 (size=12001) 2024-11-10T16:15:23,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:24,017 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e110c06b56f8496eb39857784b2417f1 2024-11-10T16:15:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5f0098a8f9574a19806fdc473b99baec is 50, key is test_row_0/B:col10/1731255322294/Put/seqid=0 2024-11-10T16:15:24,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742258_1434 (size=12001) 2024-11-10T16:15:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:24,430 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5f0098a8f9574a19806fdc473b99baec 2024-11-10T16:15:24,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/913945e3cef949dfbbfdeb2f831ea159 is 50, key is test_row_0/C:col10/1731255322294/Put/seqid=0 2024-11-10T16:15:24,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742259_1435 (size=12001) 2024-11-10T16:15:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:24,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:24,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255384528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255384533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255324522, max timestamp=1731255324542Num uniques 859; 2024-11-10T16:15:24,545 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255324522, max timestamp=1731255324542Num uniques 859; 2024-11-10T16:15:24,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255324522, max timestamp=1731255324542Num uniques 859; 2024-11-10T16:15:24,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:24,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:24,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255384596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255384597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255384601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255384654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255384654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255384713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255384713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255384724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,842 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/913945e3cef949dfbbfdeb2f831ea159 2024-11-10T16:15:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e110c06b56f8496eb39857784b2417f1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1 2024-11-10T16:15:24,849 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T16:15:24,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5f0098a8f9574a19806fdc473b99baec as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec 2024-11-10T16:15:24,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T16:15:24,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/913945e3cef949dfbbfdeb2f831ea159 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159 2024-11-10T16:15:24,857 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159, entries=150, sequenceid=80, filesize=11.7 K 2024-11-10T16:15:24,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c86cef11e7fc54de9404956c98cfb7 in 1250ms, sequenceid=80, compaction requested=true 2024-11-10T16:15:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:24,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T16:15:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T16:15:24,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T16:15:24,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-11-10T16:15:24,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.8690 sec 2024-11-10T16:15:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:24,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:24,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:24,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c42c8c0ae0b44156b31bb3c3cf011d40 is 50, key is test_row_0/A:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:24,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742260_1436 (size=14341) 2024-11-10T16:15:24,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255384912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255384914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,926 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255384934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255384934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:24,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255384943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:24,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255385023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255385033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T16:15:25,143 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:25,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T16:15:25,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:25,145 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:25,146 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:25,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:25,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255385243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255385243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255385253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255385253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255385263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,267 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c42c8c0ae0b44156b31bb3c3cf011d40 2024-11-10T16:15:25,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/f19f293a2a5c4fdfb716d060833ac15d is 50, key is test_row_0/B:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:25,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:25,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742261_1437 (size=12001) 2024-11-10T16:15:25,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:25,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:25,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255385563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255385563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:25,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:25,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/f19f293a2a5c4fdfb716d060833ac15d 2024-11-10T16:15:25,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f5264bbae6134d00ac959359553d2b0b is 50, key is test_row_0/C:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:25,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742262_1438 (size=12001) 2024-11-10T16:15:25,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:25,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:25,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255385773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255385773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255385773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:25,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:25,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:26,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:26,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255386083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255386084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:26,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f5264bbae6134d00ac959359553d2b0b 2024-11-10T16:15:26,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c42c8c0ae0b44156b31bb3c3cf011d40 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40 2024-11-10T16:15:26,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40, entries=200, sequenceid=117, filesize=14.0 K 2024-11-10T16:15:26,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/f19f293a2a5c4fdfb716d060833ac15d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d 2024-11-10T16:15:26,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T16:15:26,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f5264bbae6134d00ac959359553d2b0b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b 2024-11-10T16:15:26,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b, entries=150, sequenceid=117, filesize=11.7 K 2024-11-10T16:15:26,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 76c86cef11e7fc54de9404956c98cfb7 in 1258ms, sequenceid=117, compaction requested=true 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:26,139 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:26,139 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:26,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:26,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:26,141 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,141 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,141 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/8e966095930947d6a980e519a5484494, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=51.5 K 2024-11-10T16:15:26,141 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/2ce3aa89e71944f29b15c06603c51239, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=47.0 K 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e966095930947d6a980e519a5484494, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:26,141 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce3aa89e71944f29b15c06603c51239, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6e8a7d2eee648289f0a728d8ddb7460, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731255321112 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c655505c8b244c64b0be907f0ec549ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731255321112 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e110c06b56f8496eb39857784b2417f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731255322294 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0098a8f9574a19806fdc473b99baec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731255322294 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42c8c0ae0b44156b31bb3c3cf011d40, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:26,142 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f19f293a2a5c4fdfb716d060833ac15d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:26,158 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:26,159 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/617a2b66f93d4e4aa134cbc00fa0366b is 50, key is test_row_0/B:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:26,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742263_1439 (size=12241) 2024-11-10T16:15:26,166 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:26,167 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/5c730bc4ce784de7b89dae08ef866319 is 50, key is test_row_0/A:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:26,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742264_1440 (size=12241) 2024-11-10T16:15:26,172 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/617a2b66f93d4e4aa134cbc00fa0366b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/617a2b66f93d4e4aa134cbc00fa0366b 2024-11-10T16:15:26,175 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/5c730bc4ce784de7b89dae08ef866319 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5c730bc4ce784de7b89dae08ef866319 2024-11-10T16:15:26,177 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into 617a2b66f93d4e4aa134cbc00fa0366b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:26,177 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:26,177 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=12, startTime=1731255326139; duration=0sec 2024-11-10T16:15:26,177 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:26,177 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:26,177 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:26,178 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:26,178 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:26,179 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,179 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/327110a6d0584630b1efb2eff668723e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=47.0 K 2024-11-10T16:15:26,179 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 327110a6d0584630b1efb2eff668723e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731255321065 2024-11-10T16:15:26,179 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fce05845483e465785bd62ffb26cb826, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731255321112 2024-11-10T16:15:26,180 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into 5c730bc4ce784de7b89dae08ef866319(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:26,180 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:26,180 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=12, startTime=1731255326139; duration=0sec 2024-11-10T16:15:26,180 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:26,180 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:26,180 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 913945e3cef949dfbbfdeb2f831ea159, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731255322294 2024-11-10T16:15:26,182 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f5264bbae6134d00ac959359553d2b0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:26,191 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:26,191 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/4cbe217d31f641eb9545d903b7021d07 is 50, key is test_row_0/C:col10/1731255324595/Put/seqid=0 2024-11-10T16:15:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742265_1441 (size=12241) 2024-11-10T16:15:26,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T16:15:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:26,218 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:26,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/381803c3048e4a8d8b206cca04a3b00c is 50, key is test_row_0/A:col10/1731255324907/Put/seqid=0 2024-11-10T16:15:26,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742266_1442 (size=12001) 2024-11-10T16:15:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:26,600 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/4cbe217d31f641eb9545d903b7021d07 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/4cbe217d31f641eb9545d903b7021d07 2024-11-10T16:15:26,604 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into 4cbe217d31f641eb9545d903b7021d07(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:26,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:26,604 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=12, startTime=1731255326139; duration=0sec 2024-11-10T16:15:26,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:26,604 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:26,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/381803c3048e4a8d8b206cca04a3b00c 2024-11-10T16:15:26,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/498b635678a04673b97468b0e450ac52 is 50, key is test_row_0/B:col10/1731255324907/Put/seqid=0 2024-11-10T16:15:26,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742267_1443 (size=12001) 2024-11-10T16:15:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:26,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255326831, max timestamp=1731255326852Num uniques 859; 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255326831, max timestamp=1731255326852Num uniques 859; 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255326831, max timestamp=1731255326852Num uniques 859; 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:26,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:26,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255386909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:26,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255386910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255386911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255387023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,041 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/498b635678a04673b97468b0e450ac52 2024-11-10T16:15:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255387033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255387034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f564f15dc96e4f1dacab687499f0af9b is 50, key is test_row_0/C:col10/1731255324907/Put/seqid=0 2024-11-10T16:15:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,047 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742268_1444 (size=12001) 2024-11-10T16:15:27,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255387103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,115 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255387113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,124 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255387243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255387253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255387253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:27,453 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f564f15dc96e4f1dacab687499f0af9b 2024-11-10T16:15:27,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/381803c3048e4a8d8b206cca04a3b00c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c 2024-11-10T16:15:27,462 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T16:15:27,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/498b635678a04673b97468b0e450ac52 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52 2024-11-10T16:15:27,466 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T16:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f564f15dc96e4f1dacab687499f0af9b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b 2024-11-10T16:15:27,470 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b, entries=150, sequenceid=129, filesize=11.7 K 2024-11-10T16:15:27,470 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c86cef11e7fc54de9404956c98cfb7 in 1253ms, sequenceid=129, compaction requested=false 2024-11-10T16:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:27,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T16:15:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T16:15:27,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T16:15:27,473 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3260 sec 2024-11-10T16:15:27,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.3300 sec 2024-11-10T16:15:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:27,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:27,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:27,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/403267a06f7544c091f870d38a6c2395 is 50, key is test_row_0/A:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:27,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742269_1445 (size=14541) 2024-11-10T16:15:27,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255387588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255387588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255387590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255387703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,711 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255387713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,726 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255387713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255387913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,924 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255387933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255387933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:27,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:27,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/403267a06f7544c091f870d38a6c2395 2024-11-10T16:15:27,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/622b5220926f4050aeb0cc91d0abe2db is 50, key is test_row_0/B:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742270_1446 (size=12151) 2024-11-10T16:15:28,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/622b5220926f4050aeb0cc91d0abe2db 2024-11-10T16:15:28,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/07ed72ce614a46a58bcc2459cc240521 is 50, key is test_row_0/C:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:28,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742271_1447 (size=12151) 2024-11-10T16:15:28,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255388233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:28,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255388253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255388253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:28,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/07ed72ce614a46a58bcc2459cc240521 2024-11-10T16:15:28,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/403267a06f7544c091f870d38a6c2395 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395 2024-11-10T16:15:28,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395, entries=200, sequenceid=167, filesize=14.2 K 2024-11-10T16:15:28,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/622b5220926f4050aeb0cc91d0abe2db as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db 2024-11-10T16:15:28,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T16:15:28,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/07ed72ce614a46a58bcc2459cc240521 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521 2024-11-10T16:15:28,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521, entries=150, sequenceid=167, filesize=11.9 K 2024-11-10T16:15:28,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 76c86cef11e7fc54de9404956c98cfb7 in 864ms, sequenceid=167, compaction requested=true 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:28,435 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:28,435 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:28,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:28,436 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:28,436 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:28,436 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:28,436 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:28,436 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:28,436 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:28,436 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5c730bc4ce784de7b89dae08ef866319, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=37.9 K 2024-11-10T16:15:28,436 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/617a2b66f93d4e4aa134cbc00fa0366b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=35.5 K 2024-11-10T16:15:28,436 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c730bc4ce784de7b89dae08ef866319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:28,437 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 617a2b66f93d4e4aa134cbc00fa0366b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:28,437 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 381803c3048e4a8d8b206cca04a3b00c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731255324907 2024-11-10T16:15:28,437 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 498b635678a04673b97468b0e450ac52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731255324907 2024-11-10T16:15:28,437 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 403267a06f7544c091f870d38a6c2395, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:28,437 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 622b5220926f4050aeb0cc91d0abe2db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:28,443 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:28,444 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/6df7531c981b46568539db39d7db7b39 is 50, key is test_row_0/B:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:28,444 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:28,446 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/436e356b007240a8999fa96cb7078f1a is 50, key is test_row_0/A:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742272_1448 (size=12493) 2024-11-10T16:15:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742273_1449 (size=12493) 2024-11-10T16:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:28,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:28,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:28,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/68f42b6c065c4eaa86ed329bbaa650ee is 50, key is test_row_0/A:col10/1731255328756/Put/seqid=0 2024-11-10T16:15:28,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742274_1450 (size=14541) 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255328793, max timestamp=1731255328825Num uniques 859; 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255328793, max timestamp=1731255328825Num uniques 859; 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255328793, max timestamp=1731255328825Num uniques 859; 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:28,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:28,855 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/6df7531c981b46568539db39d7db7b39 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/6df7531c981b46568539db39d7db7b39 2024-11-10T16:15:28,855 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/436e356b007240a8999fa96cb7078f1a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/436e356b007240a8999fa96cb7078f1a 2024-11-10T16:15:28,859 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into 436e356b007240a8999fa96cb7078f1a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:28,859 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into 6df7531c981b46568539db39d7db7b39(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:28,859 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=13, startTime=1731255328435; duration=0sec 2024-11-10T16:15:28,859 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=13, startTime=1731255328435; duration=0sec 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:28,859 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:28,860 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:28,860 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:28,860 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:28,860 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/4cbe217d31f641eb9545d903b7021d07, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=35.5 K 2024-11-10T16:15:28,861 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cbe217d31f641eb9545d903b7021d07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731255324591 2024-11-10T16:15:28,861 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f564f15dc96e4f1dacab687499f0af9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731255324907 2024-11-10T16:15:28,861 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07ed72ce614a46a58bcc2459cc240521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:28,869 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:28,870 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/fd3fce0467af4e4eb68c5e163351ff99 is 50, key is test_row_0/C:col10/1731255326909/Put/seqid=0 2024-11-10T16:15:28,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742275_1451 (size=12493) 2024-11-10T16:15:28,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255388881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:28,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255388886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255388893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:28,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255389003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255389014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255389014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255389143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255389143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,151 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,151 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/68f42b6c065c4eaa86ed329bbaa650ee 2024-11-10T16:15:29,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/268f36b857764aef935a404670fe662c is 50, key is test_row_0/B:col10/1731255328756/Put/seqid=0 2024-11-10T16:15:29,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742276_1452 (size=12151) 2024-11-10T16:15:29,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/268f36b857764aef935a404670fe662c 2024-11-10T16:15:29,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/9a6a3c00c4194de2aca6ea032a52f124 is 50, key is test_row_0/C:col10/1731255328756/Put/seqid=0 2024-11-10T16:15:29,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742277_1453 (size=12151) 2024-11-10T16:15:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255389224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,282 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/fd3fce0467af4e4eb68c5e163351ff99 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fd3fce0467af4e4eb68c5e163351ff99 2024-11-10T16:15:29,286 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into fd3fce0467af4e4eb68c5e163351ff99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:29,286 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:29,286 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=13, startTime=1731255328435; duration=0sec 2024-11-10T16:15:29,286 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:29,286 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T16:15:29,313 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:29,313 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T16:15:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:29,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:29,315 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:29,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:29,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T16:15:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:29,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:29,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255389543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255389543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255389543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:29,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:29,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/9a6a3c00c4194de2aca6ea032a52f124 2024-11-10T16:15:29,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/68f42b6c065c4eaa86ed329bbaa650ee as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee 2024-11-10T16:15:29,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee, entries=200, sequenceid=177, filesize=14.2 K 2024-11-10T16:15:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/268f36b857764aef935a404670fe662c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c 2024-11-10T16:15:29,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T16:15:29,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/9a6a3c00c4194de2aca6ea032a52f124 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124 2024-11-10T16:15:29,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124, entries=150, sequenceid=177, filesize=11.9 K 2024-11-10T16:15:29,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c86cef11e7fc54de9404956c98cfb7 in 848ms, sequenceid=177, compaction requested=false 2024-11-10T16:15:29,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:29,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T16:15:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:29,621 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:29,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:29,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f57849b1ab8d445a9af14bba9d66c45b is 50, key is test_row_0/A:col10/1731255328891/Put/seqid=0 2024-11-10T16:15:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:29,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742278_1454 (size=12151) 2024-11-10T16:15:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:30,035 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f57849b1ab8d445a9af14bba9d66c45b 2024-11-10T16:15:30,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/9fba5e6aa4fd4cc09752733d96bb0eb8 is 50, key is test_row_0/B:col10/1731255328891/Put/seqid=0 2024-11-10T16:15:30,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742279_1455 (size=12151) 2024-11-10T16:15:30,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:30,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:30,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255390086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255390088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255390090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,104 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255390193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255390203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255390213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255390413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255390423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255390433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,447 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/9fba5e6aa4fd4cc09752733d96bb0eb8 2024-11-10T16:15:30,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:30,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/303b9842b7c54363b7ab11a21b880dc9 is 50, key is test_row_0/C:col10/1731255328891/Put/seqid=0 2024-11-10T16:15:30,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742280_1456 (size=12151) 2024-11-10T16:15:30,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255390733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255390743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255390754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:30,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/303b9842b7c54363b7ab11a21b880dc9 2024-11-10T16:15:30,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/f57849b1ab8d445a9af14bba9d66c45b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b 2024-11-10T16:15:30,866 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T16:15:30,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/9fba5e6aa4fd4cc09752733d96bb0eb8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8 2024-11-10T16:15:30,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T16:15:30,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/303b9842b7c54363b7ab11a21b880dc9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9 2024-11-10T16:15:30,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T16:15:30,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 76c86cef11e7fc54de9404956c98cfb7 in 1253ms, sequenceid=216, compaction requested=true 2024-11-10T16:15:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:30,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:30,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T16:15:30,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T16:15:30,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T16:15:30,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5600 sec 2024-11-10T16:15:30,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5660 sec 2024-11-10T16:15:31,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:31,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:31,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:31,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/3bc9974fc7a646c48927046293b88759 is 50, key is test_row_0/A:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:31,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742281_1457 (size=14541) 2024-11-10T16:15:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255391322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255331307, max timestamp=1731255331324Num uniques 859; 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255331307, max timestamp=1731255331324Num uniques 859; 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255331307, max timestamp=1731255331324Num uniques 859; 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:31,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:31,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255391390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255391403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255391433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T16:15:31,464 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:31,464 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:31,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T16:15:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:31,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:31,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:31,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:31,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255391513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255391513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:31,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T16:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255391653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/3bc9974fc7a646c48927046293b88759 2024-11-10T16:15:31,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/357bf83e2e584a68829dce3e288d7149 is 50, key is test_row_0/B:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:31,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742282_1458 (size=12151) 2024-11-10T16:15:31,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255391733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255391733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:31,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T16:15:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:31,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T16:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:31,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:31,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:31,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255391973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:31,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255392043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255392044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T16:15:32,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/357bf83e2e584a68829dce3e288d7149 2024-11-10T16:15:32,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:32,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/c31c4dbd1d884834af124c9e53fa08e3 is 50, key is test_row_0/C:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742283_1459 (size=12151) 2024-11-10T16:15:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:32,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/c31c4dbd1d884834af124c9e53fa08e3 2024-11-10T16:15:32,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/3bc9974fc7a646c48927046293b88759 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759 2024-11-10T16:15:32,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759, entries=200, sequenceid=227, filesize=14.2 K 2024-11-10T16:15:32,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/357bf83e2e584a68829dce3e288d7149 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149 2024-11-10T16:15:32,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T16:15:32,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/c31c4dbd1d884834af124c9e53fa08e3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3 2024-11-10T16:15:32,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3, entries=150, sequenceid=227, filesize=11.9 K 2024-11-10T16:15:32,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c86cef11e7fc54de9404956c98cfb7 in 857ms, sequenceid=227, compaction requested=true 2024-11-10T16:15:32,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:32,115 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:32,115 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:32,116 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53726 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:32,116 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:32,116 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,116 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/436e356b007240a8999fa96cb7078f1a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=52.5 K 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 436e356b007240a8999fa96cb7078f1a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:32,117 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,117 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/6df7531c981b46568539db39d7db7b39, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=47.8 K 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68f42b6c065c4eaa86ed329bbaa650ee, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731255327570 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 6df7531c981b46568539db39d7db7b39, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57849b1ab8d445a9af14bba9d66c45b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731255328849 2024-11-10T16:15:32,117 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 268f36b857764aef935a404670fe662c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731255327570 2024-11-10T16:15:32,118 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bc9974fc7a646c48927046293b88759, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:32,118 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fba5e6aa4fd4cc09752733d96bb0eb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731255328849 2024-11-10T16:15:32,121 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 357bf83e2e584a68829dce3e288d7149, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:32,131 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:32,132 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:32,132 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/acefa7cf67f143dfb28d6731cec4d5a1 is 50, key is test_row_0/B:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:32,132 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c758885e32b94dbf953ffc1307a04677 is 50, key is test_row_0/A:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:32,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742285_1461 (size=12629) 2024-11-10T16:15:32,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742284_1460 (size=12629) 2024-11-10T16:15:32,142 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c758885e32b94dbf953ffc1307a04677 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c758885e32b94dbf953ffc1307a04677 2024-11-10T16:15:32,146 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into c758885e32b94dbf953ffc1307a04677(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:32,146 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:32,146 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=12, startTime=1731255332115; duration=0sec 2024-11-10T16:15:32,146 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:32,146 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:32,147 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:32,148 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:32,148 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:32,148 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,148 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fd3fce0467af4e4eb68c5e163351ff99, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=47.8 K 2024-11-10T16:15:32,149 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd3fce0467af4e4eb68c5e163351ff99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731255326894 2024-11-10T16:15:32,149 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a6a3c00c4194de2aca6ea032a52f124, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731255327570 2024-11-10T16:15:32,149 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 303b9842b7c54363b7ab11a21b880dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731255328849 2024-11-10T16:15:32,150 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31c4dbd1d884834af124c9e53fa08e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:32,160 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:32,160 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f9547ab0787f42f0bcf3d27ddd759c47 is 50, key is test_row_0/C:col10/1731255330072/Put/seqid=0 2024-11-10T16:15:32,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742286_1462 (size=12629) 2024-11-10T16:15:32,172 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/f9547ab0787f42f0bcf3d27ddd759c47 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f9547ab0787f42f0bcf3d27ddd759c47 2024-11-10T16:15:32,176 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into f9547ab0787f42f0bcf3d27ddd759c47(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:32,176 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:32,176 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=12, startTime=1731255332115; duration=0sec 2024-11-10T16:15:32,176 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:32,176 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:32,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T16:15:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,232 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:32,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:32,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c56ab939aab248619e8bf8b0d82b619e is 50, key is test_row_0/A:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742287_1463 (size=12301) 2024-11-10T16:15:32,243 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c56ab939aab248619e8bf8b0d82b619e 2024-11-10T16:15:32,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d124d713f7764015973566de445c50af is 50, key is test_row_0/B:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:32,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742288_1464 (size=12301) 2024-11-10T16:15:32,254 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d124d713f7764015973566de445c50af 2024-11-10T16:15:32,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e3e92c43153742bb9905516392b76b35 is 50, key is test_row_0/C:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:32,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742289_1465 (size=12301) 2024-11-10T16:15:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:32,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:32,542 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/acefa7cf67f143dfb28d6731cec4d5a1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/acefa7cf67f143dfb28d6731cec4d5a1 2024-11-10T16:15:32,547 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into acefa7cf67f143dfb28d6731cec4d5a1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:32,547 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:32,547 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=12, startTime=1731255332115; duration=0sec 2024-11-10T16:15:32,547 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:32,547 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255392563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255392563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255392569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:32,664 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e3e92c43153742bb9905516392b76b35 2024-11-10T16:15:32,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c56ab939aab248619e8bf8b0d82b619e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e 2024-11-10T16:15:32,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T16:15:32,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d124d713f7764015973566de445c50af as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af 2024-11-10T16:15:32,676 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T16:15:32,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e3e92c43153742bb9905516392b76b35 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35 2024-11-10T16:15:32,680 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T16:15:32,681 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 76c86cef11e7fc54de9404956c98cfb7 in 449ms, sequenceid=263, compaction requested=false 2024-11-10T16:15:32,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:32,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:32,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T16:15:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T16:15:32,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T16:15:32,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-10T16:15:32,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.2210 sec 2024-11-10T16:15:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:32,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:32,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:32,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c871a32a398d4f7ea28710b6937729ef is 50, key is test_row_1/A:col10/1731255332686/Put/seqid=0 2024-11-10T16:15:32,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742290_1466 (size=12297) 2024-11-10T16:15:32,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c871a32a398d4f7ea28710b6937729ef 2024-11-10T16:15:32,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/7754bb2acd7c4004b019daaa327b5224 is 50, key is test_row_1/B:col10/1731255332686/Put/seqid=0 2024-11-10T16:15:32,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742291_1467 (size=9857) 2024-11-10T16:15:32,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/7754bb2acd7c4004b019daaa327b5224 2024-11-10T16:15:32,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b58ddfa39b2b47d0b56de67fcaa2ae30 is 50, key is test_row_1/C:col10/1731255332686/Put/seqid=0 2024-11-10T16:15:32,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742292_1468 (size=9857) 2024-11-10T16:15:32,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255392826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:32,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:32,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255392943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:32,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b58ddfa39b2b47d0b56de67fcaa2ae30 2024-11-10T16:15:33,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/c871a32a398d4f7ea28710b6937729ef as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef 2024-11-10T16:15:33,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef, entries=150, sequenceid=276, filesize=12.0 K 2024-11-10T16:15:33,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/7754bb2acd7c4004b019daaa327b5224 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224 2024-11-10T16:15:33,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224, entries=100, sequenceid=276, filesize=9.6 K 2024-11-10T16:15:33,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/b58ddfa39b2b47d0b56de67fcaa2ae30 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30 2024-11-10T16:15:33,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30, entries=100, sequenceid=276, filesize=9.6 K 2024-11-10T16:15:33,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 76c86cef11e7fc54de9404956c98cfb7 in 455ms, sequenceid=276, compaction requested=true 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:33,143 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:33,143 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:33,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:33,144 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:33,144 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:33,144 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:33,144 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:33,144 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,144 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,144 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/acefa7cf67f143dfb28d6731cec4d5a1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=34.0 K 2024-11-10T16:15:33,144 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c758885e32b94dbf953ffc1307a04677, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=36.4 K 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting acefa7cf67f143dfb28d6731cec4d5a1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c758885e32b94dbf953ffc1307a04677, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d124d713f7764015973566de445c50af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731255331384 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c56ab939aab248619e8bf8b0d82b619e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731255331384 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c871a32a398d4f7ea28710b6937729ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255332537 2024-11-10T16:15:33,145 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7754bb2acd7c4004b019daaa327b5224, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255332537 2024-11-10T16:15:33,152 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:33,152 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:33,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/91c416f768814c4c91b2dd4e9fc9638b is 50, key is test_row_0/A:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:33,152 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/7f29bd99930d48af98980f6eeccc40fc is 50, key is test_row_0/B:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742293_1469 (size=12881) 2024-11-10T16:15:33,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742294_1470 (size=12881) 2024-11-10T16:15:33,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:33,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:33,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:33,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/89f935236c7b49af9f7deb7318810381 is 50, key is test_row_0/A:col10/1731255332817/Put/seqid=0 2024-11-10T16:15:33,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742295_1471 (size=14741) 2024-11-10T16:15:33,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/89f935236c7b49af9f7deb7318810381 2024-11-10T16:15:33,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c3ae9595656349d8b4f45c9720ee9a9e is 50, key is test_row_0/B:col10/1731255332817/Put/seqid=0 2024-11-10T16:15:33,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742296_1472 (size=12301) 2024-11-10T16:15:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255393186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255393190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255393201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255393313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255393313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255393323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255393533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255393534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255393543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,561 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/7f29bd99930d48af98980f6eeccc40fc as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7f29bd99930d48af98980f6eeccc40fc 2024-11-10T16:15:33,561 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/91c416f768814c4c91b2dd4e9fc9638b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/91c416f768814c4c91b2dd4e9fc9638b 2024-11-10T16:15:33,565 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into 91c416f768814c4c91b2dd4e9fc9638b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:33,565 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into 7f29bd99930d48af98980f6eeccc40fc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:33,565 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=13, startTime=1731255333143; duration=0sec 2024-11-10T16:15:33,565 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=13, startTime=1731255333143; duration=0sec 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:33,565 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:33,566 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:33,566 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:33,566 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,566 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f9547ab0787f42f0bcf3d27ddd759c47, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=34.0 K 2024-11-10T16:15:33,567 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f9547ab0787f42f0bcf3d27ddd759c47, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731255330072 2024-11-10T16:15:33,567 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e92c43153742bb9905516392b76b35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731255331384 2024-11-10T16:15:33,567 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b58ddfa39b2b47d0b56de67fcaa2ae30, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255332537 2024-11-10T16:15:33,573 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#399 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:33,574 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/74f4ef7def6547d8a5860629e7676c52 is 50, key is test_row_0/C:col10/1731255331402/Put/seqid=0 2024-11-10T16:15:33,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742297_1473 (size=12881) 2024-11-10T16:15:33,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c3ae9595656349d8b4f45c9720ee9a9e 2024-11-10T16:15:33,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255393584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255393584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,590 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,591 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:33,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e83aa8ccc1824b5a8d6a7409c7f4c323 is 50, key is test_row_0/C:col10/1731255332817/Put/seqid=0 2024-11-10T16:15:33,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742298_1474 (size=12301) 2024-11-10T16:15:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T16:15:33,623 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:33,624 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T16:15:33,625 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:33,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:33,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:33,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T16:15:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255393853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255393853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255393863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:33,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T16:15:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:33,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:33,982 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/74f4ef7def6547d8a5860629e7676c52 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/74f4ef7def6547d8a5860629e7676c52 2024-11-10T16:15:33,986 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into 74f4ef7def6547d8a5860629e7676c52(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:33,986 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:33,986 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=13, startTime=1731255333143; duration=0sec 2024-11-10T16:15:33,986 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:33,986 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:34,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e83aa8ccc1824b5a8d6a7409c7f4c323 2024-11-10T16:15:34,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/89f935236c7b49af9f7deb7318810381 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381 2024-11-10T16:15:34,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381, entries=200, sequenceid=297, filesize=14.4 K 2024-11-10T16:15:34,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c3ae9595656349d8b4f45c9720ee9a9e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e 2024-11-10T16:15:34,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T16:15:34,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/e83aa8ccc1824b5a8d6a7409c7f4c323 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323 2024-11-10T16:15:34,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323, entries=150, sequenceid=297, filesize=12.0 K 2024-11-10T16:15:34,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 76c86cef11e7fc54de9404956c98cfb7 in 853ms, sequenceid=297, compaction requested=false 2024-11-10T16:15:34,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:34,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T16:15:34,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:34,084 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:34,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:34,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/bedacef4bf5b40df8f86bf97fa947a94 is 50, key is test_row_0/A:col10/1731255333183/Put/seqid=0 2024-11-10T16:15:34,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742299_1475 (size=12301) 2024-11-10T16:15:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:34,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255334404, max timestamp=1731255334435Num uniques 859; 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255334404, max timestamp=1731255334435Num uniques 859; 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255334404, max timestamp=1731255334435Num uniques 859; 2024-11-10T16:15:34,439 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:34,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/bedacef4bf5b40df8f86bf97fa947a94 2024-11-10T16:15:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/723e55acbea043ebb275fa97e22802c9 is 50, key is test_row_0/B:col10/1731255333183/Put/seqid=0 2024-11-10T16:15:34,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742300_1476 (size=12301) 2024-11-10T16:15:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255394504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255394516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255394517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255394623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255394633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255394643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:34,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255394843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255394843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255394863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:34,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:34,911 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/723e55acbea043ebb275fa97e22802c9 2024-11-10T16:15:34,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/57454234f3a44352bdee5ea809396ba0 is 50, key is test_row_0/C:col10/1731255333183/Put/seqid=0 2024-11-10T16:15:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742301_1477 (size=12301) 2024-11-10T16:15:35,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255395153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255395153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255395183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,322 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/57454234f3a44352bdee5ea809396ba0 2024-11-10T16:15:35,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/bedacef4bf5b40df8f86bf97fa947a94 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94 2024-11-10T16:15:35,330 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T16:15:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/723e55acbea043ebb275fa97e22802c9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9 2024-11-10T16:15:35,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T16:15:35,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/57454234f3a44352bdee5ea809396ba0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0 2024-11-10T16:15:35,338 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T16:15:35,339 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 76c86cef11e7fc54de9404956c98cfb7 in 1255ms, sequenceid=311, compaction requested=true 2024-11-10T16:15:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T16:15:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T16:15:35,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T16:15:35,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7150 sec 2024-11-10T16:15:35,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7190 sec 2024-11-10T16:15:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:35,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:35,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:35,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e8ccfebfd5714cd49cdfe39dd41da0f4 is 50, key is test_row_0/A:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742302_1478 (size=14741) 2024-11-10T16:15:35,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255395647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,663 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255395657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255395673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255395673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255395703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T16:15:35,774 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:35,774 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:35,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-10T16:15:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:35,776 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:35,777 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:35,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:35,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255395773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:35,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255395774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:35,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:35,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:35,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:35,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:35,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:35,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:35,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255395993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255395993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e8ccfebfd5714cd49cdfe39dd41da0f4 2024-11-10T16:15:36,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/db8ce9051b0b46cf9a857630b66af35a is 50, key is test_row_0/B:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:36,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742303_1479 (size=12301) 2024-11-10T16:15:36,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:36,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255396313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255396313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:36,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/db8ce9051b0b46cf9a857630b66af35a 2024-11-10T16:15:36,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/165bfd4a5f6443ceb51fe693fdb54b74 is 50, key is test_row_0/C:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742304_1480 (size=12301) 2024-11-10T16:15:36,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32972 deadline: 1731255396693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,705 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33010 deadline: 1731255396703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,717 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32996 deadline: 1731255396733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,748 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 76c86cef11e7fc54de9404956c98cfb7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255396833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:36,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:36,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/165bfd4a5f6443ceb51fe693fdb54b74 2024-11-10T16:15:36,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:36,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/e8ccfebfd5714cd49cdfe39dd41da0f4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4 2024-11-10T16:15:36,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4, entries=200, sequenceid=347, filesize=14.4 K 2024-11-10T16:15:36,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/db8ce9051b0b46cf9a857630b66af35a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a 2024-11-10T16:15:36,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a, entries=150, sequenceid=347, filesize=12.0 K 2024-11-10T16:15:36,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/165bfd4a5f6443ceb51fe693fdb54b74 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74 2024-11-10T16:15:36,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74, entries=150, sequenceid=347, filesize=12.0 K 2024-11-10T16:15:36,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 76c86cef11e7fc54de9404956c98cfb7 in 1250ms, sequenceid=347, compaction requested=true 2024-11-10T16:15:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:36,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:36,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:36,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:36,864 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:36,864 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:36,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:36,864 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:36,864 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:36,864 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:36,865 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,865 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:36,865 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7f29bd99930d48af98980f6eeccc40fc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=48.6 K 2024-11-10T16:15:36,865 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,865 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/91c416f768814c4c91b2dd4e9fc9638b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=53.4 K 2024-11-10T16:15:36,865 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f29bd99930d48af98980f6eeccc40fc, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255331402 2024-11-10T16:15:36,865 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91c416f768814c4c91b2dd4e9fc9638b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255331402 2024-11-10T16:15:36,865 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ae9595656349d8b4f45c9720ee9a9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731255332817 2024-11-10T16:15:36,865 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f935236c7b49af9f7deb7318810381, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731255332774 2024-11-10T16:15:36,866 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting bedacef4bf5b40df8f86bf97fa947a94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731255333176 2024-11-10T16:15:36,866 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 723e55acbea043ebb275fa97e22802c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731255333176 2024-11-10T16:15:36,866 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8ccfebfd5714cd49cdfe39dd41da0f4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:36,866 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting db8ce9051b0b46cf9a857630b66af35a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:36,873 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:36,873 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:36,874 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c753768136f84ee58f250824f71337d1 is 50, key is test_row_0/B:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:36,874 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/7f0124add6e5448cbc0d8f6f5c3bd770 is 50, key is test_row_0/A:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:36,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742305_1481 (size=13017) 2024-11-10T16:15:36,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742306_1482 (size=13017) 2024-11-10T16:15:36,883 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/c753768136f84ee58f250824f71337d1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c753768136f84ee58f250824f71337d1 2024-11-10T16:15:36,887 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into c753768136f84ee58f250824f71337d1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:36,887 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:36,887 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=12, startTime=1731255336863; duration=0sec 2024-11-10T16:15:36,887 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:36,887 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:36,887 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:36,888 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:36,888 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:36,888 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:36,888 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/74f4ef7def6547d8a5860629e7676c52, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=48.6 K 2024-11-10T16:15:36,888 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 74f4ef7def6547d8a5860629e7676c52, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731255331402 2024-11-10T16:15:36,889 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting e83aa8ccc1824b5a8d6a7409c7f4c323, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731255332817 2024-11-10T16:15:36,889 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 57454234f3a44352bdee5ea809396ba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731255333176 2024-11-10T16:15:36,889 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 165bfd4a5f6443ceb51fe693fdb54b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:36,896 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:36,897 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/14b2f4abcbfe4c3fbcb939e737904c3a is 50, key is test_row_0/C:col10/1731255335611/Put/seqid=0 2024-11-10T16:15:36,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742307_1483 (size=13017) 2024-11-10T16:15:36,905 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/14b2f4abcbfe4c3fbcb939e737904c3a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/14b2f4abcbfe4c3fbcb939e737904c3a 2024-11-10T16:15:36,909 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into 14b2f4abcbfe4c3fbcb939e737904c3a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:36,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:36,909 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=12, startTime=1731255336864; duration=0sec 2024-11-10T16:15:36,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:36,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:37,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-10T16:15:37,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:37,001 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:37,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/00ba6771cdfa4836ad345a592bd1dfd5 is 50, key is test_row_0/A:col10/1731255335655/Put/seqid=0 2024-11-10T16:15:37,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742308_1484 (size=12301) 2024-11-10T16:15:37,009 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/00ba6771cdfa4836ad345a592bd1dfd5 2024-11-10T16:15:37,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/728b295f29934409ad5b188b7d3b735b is 50, key is test_row_0/B:col10/1731255335655/Put/seqid=0 2024-11-10T16:15:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742309_1485 (size=12301) 2024-11-10T16:15:37,283 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/7f0124add6e5448cbc0d8f6f5c3bd770 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/7f0124add6e5448cbc0d8f6f5c3bd770 2024-11-10T16:15:37,287 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into 7f0124add6e5448cbc0d8f6f5c3bd770(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:37,287 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:37,287 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=12, startTime=1731255336863; duration=0sec 2024-11-10T16:15:37,288 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:37,288 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:37,422 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/728b295f29934409ad5b188b7d3b735b 2024-11-10T16:15:37,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/7fff2ad3b6674cc490627b7d3260ed81 is 50, key is test_row_0/C:col10/1731255335655/Put/seqid=0 2024-11-10T16:15:37,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742310_1486 (size=12301) 2024-11-10T16:15:37,839 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/7fff2ad3b6674cc490627b7d3260ed81 2024-11-10T16:15:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/00ba6771cdfa4836ad345a592bd1dfd5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5 2024-11-10T16:15:37,848 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T16:15:37,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/728b295f29934409ad5b188b7d3b735b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b 2024-11-10T16:15:37,852 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T16:15:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/7fff2ad3b6674cc490627b7d3260ed81 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81 2024-11-10T16:15:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:37,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:37,858 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81, entries=150, sequenceid=359, filesize=12.0 K 2024-11-10T16:15:37,859 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 76c86cef11e7fc54de9404956c98cfb7 in 859ms, sequenceid=359, compaction requested=false 2024-11-10T16:15:37,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:37,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:37,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-10T16:15:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-10T16:15:37,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T16:15:37,863 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0840 sec 2024-11-10T16:15:37,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0890 sec 2024-11-10T16:15:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:37,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:37,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/5d2c3f2334b2400c871265ea20388260 is 50, key is test_row_0/A:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:37,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742311_1487 (size=17181) 2024-11-10T16:15:37,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/5d2c3f2334b2400c871265ea20388260 2024-11-10T16:15:37,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d4ec193f23e346a1a88a2405d009348b is 50, key is test_row_0/B:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:37,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742312_1488 (size=12301) 2024-11-10T16:15:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T16:15:37,923 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:37,923 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-10T16:15:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:37,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:37,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:37,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255337961, max timestamp=1731255337981Num uniques 859; 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255337961, max timestamp=1731255337981Num uniques 859; 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255337961, max timestamp=1731255337981Num uniques 859; 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:37,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:38,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255398077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255398093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255398203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255398203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:38,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d4ec193f23e346a1a88a2405d009348b 2024-11-10T16:15:38,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/da833c9015f249fdb31a93ba317bfd4c is 50, key is test_row_0/C:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:38,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742313_1489 (size=12301) 2024-11-10T16:15:38,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255398413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33028 deadline: 1731255398413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,421 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:38,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/da833c9015f249fdb31a93ba317bfd4c 2024-11-10T16:15:38,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/5d2c3f2334b2400c871265ea20388260 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260 2024-11-10T16:15:38,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260, entries=250, sequenceid=370, filesize=16.8 K 2024-11-10T16:15:38,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d4ec193f23e346a1a88a2405d009348b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b 2024-11-10T16:15:38,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b, entries=150, sequenceid=370, filesize=12.0 K 2024-11-10T16:15:38,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/da833c9015f249fdb31a93ba317bfd4c as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c 2024-11-10T16:15:38,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c, entries=150, sequenceid=370, filesize=12.0 K 2024-11-10T16:15:38,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 76c86cef11e7fc54de9404956c98cfb7 in 855ms, sequenceid=370, compaction requested=true 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 76c86cef11e7fc54de9404956c98cfb7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:38,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:38,725 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:38,725 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:38,726 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:38,726 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:38,726 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/B is initiating minor compaction (all files) 2024-11-10T16:15:38,726 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/A is initiating minor compaction (all files) 2024-11-10T16:15:38,726 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/B in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,726 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/A in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,726 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c753768136f84ee58f250824f71337d1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=36.7 K 2024-11-10T16:15:38,726 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/7f0124add6e5448cbc0d8f6f5c3bd770, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=41.5 K 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting c753768136f84ee58f250824f71337d1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f0124add6e5448cbc0d8f6f5c3bd770, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ba6771cdfa4836ad345a592bd1dfd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731255335639 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 728b295f29934409ad5b188b7d3b735b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731255335639 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d2c3f2334b2400c871265ea20388260, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731255337856 2024-11-10T16:15:38,727 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d4ec193f23e346a1a88a2405d009348b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731255337856 2024-11-10T16:15:38,734 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:38,734 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d9317deef2504e0eb579bd8c45832405 is 50, key is test_row_0/B:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:38,734 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:38,735 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/aaaa08393e214c308bfffd4ee43f5b20 is 50, key is test_row_0/A:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:38,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255338054, max timestamp=1731255338733Num uniques 1018; 2024-11-10T16:15:38,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255338054, max timestamp=1731255338733Num uniques 1018; 2024-11-10T16:15:38,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731255338054, max timestamp=1731255338733Num uniques 1018; 2024-11-10T16:15:38,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T16:15:38,738 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T16:15:38,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T16:15:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742315_1491 (size=13119) 2024-11-10T16:15:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742314_1490 (size=13119) 2024-11-10T16:15:38,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:38,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=60.38 KB heapSize=165 KB 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:38,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/eb68fa1977264044928d9998bf5eb422 is 50, key is test_row_0/A:col10/1731255338092/Put/seqid=0 2024-11-10T16:15:38,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742316_1492 (size=17181) 2024-11-10T16:15:38,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32982 deadline: 1731255398792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 , the old value is region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., hostname=8e3de9160f9f,44105,1731255200149, seqNum=2 because the exception is null or not the one we care about 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255338781, max timestamp=1731255338798Num uniques 859; 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255338781, max timestamp=1731255338798Num uniques 859; 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731255338781, max timestamp=1731255338798Num uniques 859; 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:38,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T16:15:38,823 INFO [Thread-1897 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1897. 2024-11-10T16:15:38,823 DEBUG [Thread-1897 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,823 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,823 DEBUG [Thread-1897 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,824 INFO [Thread-1888 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1888. 2024-11-10T16:15:38,824 DEBUG [Thread-1888 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,824 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,824 DEBUG [Thread-1888 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,826 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-10T16:15:38,826 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,826 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,826 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,826 INFO [Thread-1895 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1895. 2024-11-10T16:15:38,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,826 DEBUG [Thread-1895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,826 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,827 DEBUG [Thread-1895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,828 INFO [Thread-1890 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1890. 2024-11-10T16:15:38,828 DEBUG [Thread-1890 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,828 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,828 DEBUG [Thread-1890 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,829 INFO [Thread-1884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1884. 2024-11-10T16:15:38,829 DEBUG [Thread-1884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,829 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,829 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,829 INFO [Thread-1899 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1899. 2024-11-10T16:15:38,829 DEBUG [Thread-1899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,829 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,829 DEBUG [Thread-1899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,830 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,843 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-10T16:15:38,843 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,843 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,843 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,844 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-10T16:15:38,844 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,844 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,844 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:38,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:38,905 INFO [Thread-1892 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1892. 2024-11-10T16:15:38,905 DEBUG [Thread-1892 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:15:38,905 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,905 DEBUG [Thread-1892 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:38,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:38,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:39,148 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/d9317deef2504e0eb579bd8c45832405 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d9317deef2504e0eb579bd8c45832405 2024-11-10T16:15:39,148 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/aaaa08393e214c308bfffd4ee43f5b20 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/aaaa08393e214c308bfffd4ee43f5b20 2024-11-10T16:15:39,151 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/B of 76c86cef11e7fc54de9404956c98cfb7 into d9317deef2504e0eb579bd8c45832405(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:39,151 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/A of 76c86cef11e7fc54de9404956c98cfb7 into aaaa08393e214c308bfffd4ee43f5b20(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:39,152 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/A, priority=13, startTime=1731255338725; duration=0sec 2024-11-10T16:15:39,152 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/B, priority=13, startTime=1731255338725; duration=0sec 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:A 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:B 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:39,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,152 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:39,153 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 76c86cef11e7fc54de9404956c98cfb7/C is initiating minor compaction (all files) 2024-11-10T16:15:39,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,153 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 76c86cef11e7fc54de9404956c98cfb7/C in TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,153 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/14b2f4abcbfe4c3fbcb939e737904c3a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp, totalSize=36.7 K 2024-11-10T16:15:39,153 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14b2f4abcbfe4c3fbcb939e737904c3a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731255334492 2024-11-10T16:15:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,153 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fff2ad3b6674cc490627b7d3260ed81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731255335639 2024-11-10T16:15:39,154 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting da833c9015f249fdb31a93ba317bfd4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731255337856 2024-11-10T16:15:39,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/eb68fa1977264044928d9998bf5eb422 2024-11-10T16:15:39,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5b88800d445e475c83a18d5dae8bef5e is 50, key is test_row_0/B:col10/1731255338092/Put/seqid=0 2024-11-10T16:15:39,160 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 76c86cef11e7fc54de9404956c98cfb7#C#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:39,161 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/6aab20ca3cda452f965b3c5574f95454 is 50, key is test_row_0/C:col10/1731255337868/Put/seqid=0 2024-11-10T16:15:39,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742317_1493 (size=12301) 2024-11-10T16:15:39,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742318_1494 (size=13119) 2024-11-10T16:15:39,168 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/6aab20ca3cda452f965b3c5574f95454 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/6aab20ca3cda452f965b3c5574f95454 2024-11-10T16:15:39,172 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 76c86cef11e7fc54de9404956c98cfb7/C of 76c86cef11e7fc54de9404956c98cfb7 into 6aab20ca3cda452f965b3c5574f95454(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:39,172 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:39,172 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7., storeName=76c86cef11e7fc54de9404956c98cfb7/C, priority=13, startTime=1731255338725; duration=0sec 2024-11-10T16:15:39,172 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:39,172 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 76c86cef11e7fc54de9404956c98cfb7:C 2024-11-10T16:15:39,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5b88800d445e475c83a18d5dae8bef5e 2024-11-10T16:15:39,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/09f149390f6c4fbe8e2290a2f74b49c3 is 50, key is test_row_0/C:col10/1731255338092/Put/seqid=0 2024-11-10T16:15:39,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742319_1495 (size=12301) 2024-11-10T16:15:39,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:39,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. as already flushing 2024-11-10T16:15:39,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:39,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:39,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/09f149390f6c4fbe8e2290a2f74b49c3 2024-11-10T16:15:39,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/eb68fa1977264044928d9998bf5eb422 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/eb68fa1977264044928d9998bf5eb422 2024-11-10T16:15:39,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/eb68fa1977264044928d9998bf5eb422, entries=250, sequenceid=413, filesize=16.8 K 2024-11-10T16:15:39,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/5b88800d445e475c83a18d5dae8bef5e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5b88800d445e475c83a18d5dae8bef5e 2024-11-10T16:15:39,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5b88800d445e475c83a18d5dae8bef5e, entries=150, sequenceid=413, filesize=12.0 K 2024-11-10T16:15:39,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/09f149390f6c4fbe8e2290a2f74b49c3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/09f149390f6c4fbe8e2290a2f74b49c3 2024-11-10T16:15:39,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/09f149390f6c4fbe8e2290a2f74b49c3, entries=150, sequenceid=413, filesize=12.0 K 2024-11-10T16:15:39,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=100.63 KB/103050 for 76c86cef11e7fc54de9404956c98cfb7 in 1239ms, sequenceid=413, compaction requested=false 2024-11-10T16:15:39,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:40,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-10T16:15:40,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:40,070 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 76c86cef11e7fc54de9404956c98cfb7 3/3 column families, dataSize=100.63 KB heapSize=276 KB 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=A 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=B 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 76c86cef11e7fc54de9404956c98cfb7, store=C 2024-11-10T16:15:40,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/dd626579e08d4a36bfbd8294c604e82a is 50, key is test_row_0/A:col10/1731255338828/Put/seqid=0 2024-11-10T16:15:40,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742320_1496 (size=12301) 2024-11-10T16:15:40,477 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/dd626579e08d4a36bfbd8294c604e82a 2024-11-10T16:15:40,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/3aff85f157224520b90189274fe74c5d is 50, key is test_row_0/B:col10/1731255338828/Put/seqid=0 2024-11-10T16:15:40,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742321_1497 (size=12301) 2024-11-10T16:15:40,887 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/3aff85f157224520b90189274fe74c5d 2024-11-10T16:15:40,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/092f1d65ad61495b83685d9f05fdf851 is 50, key is test_row_0/C:col10/1731255338828/Put/seqid=0 2024-11-10T16:15:40,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742322_1498 (size=12301) 2024-11-10T16:15:41,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/092f1d65ad61495b83685d9f05fdf851 2024-11-10T16:15:41,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/A/dd626579e08d4a36bfbd8294c604e82a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/dd626579e08d4a36bfbd8294c604e82a 2024-11-10T16:15:41,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/dd626579e08d4a36bfbd8294c604e82a, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T16:15:41,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/B/3aff85f157224520b90189274fe74c5d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/3aff85f157224520b90189274fe74c5d 2024-11-10T16:15:41,306 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/3aff85f157224520b90189274fe74c5d, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T16:15:41,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/.tmp/C/092f1d65ad61495b83685d9f05fdf851 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/092f1d65ad61495b83685d9f05fdf851 2024-11-10T16:15:41,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/092f1d65ad61495b83685d9f05fdf851, entries=150, sequenceid=446, filesize=12.0 K 2024-11-10T16:15:41,310 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~275.95 KB/282576, currentSize=0 B/0 for 76c86cef11e7fc54de9404956c98cfb7 in 1241ms, sequenceid=446, compaction requested=true 2024-11-10T16:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 76c86cef11e7fc54de9404956c98cfb7: 2024-11-10T16:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-10T16:15:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-10T16:15:41,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T16:15:41,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3850 sec 2024-11-10T16:15:41,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 3.3890 sec 2024-11-10T16:15:42,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-10T16:15:42,093 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:42,093 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3143 rows 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1056 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3164 rows 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1052 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3153 rows 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1050 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3149 rows 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1051 2024-11-10T16:15:42,094 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3152 rows 2024-11-10T16:15:42,094 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:15:42,094 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:15:42,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:42,094 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:42,094 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:15:42,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:15:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T16:15:42,097 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255342097"}]},"ts":"1731255342097"} 2024-11-10T16:15:42,099 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:15:42,099 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:15:42,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:15:42,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, UNASSIGN}] 2024-11-10T16:15:42,101 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, UNASSIGN 2024-11-10T16:15:42,102 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=76c86cef11e7fc54de9404956c98cfb7, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:42,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, UNASSIGN because future has completed 2024-11-10T16:15:42,103 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:15:42,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T16:15:42,255 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:42,255 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:15:42,255 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 76c86cef11e7fc54de9404956c98cfb7, disabling compactions & flushes 2024-11-10T16:15:42,255 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:42,255 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:42,255 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. after waiting 0 ms 2024-11-10T16:15:42,255 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:42,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/8e966095930947d6a980e519a5484494, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5c730bc4ce784de7b89dae08ef866319, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/436e356b007240a8999fa96cb7078f1a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c758885e32b94dbf953ffc1307a04677, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/91c416f768814c4c91b2dd4e9fc9638b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/7f0124add6e5448cbc0d8f6f5c3bd770, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260] to archive 2024-11-10T16:15:42,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:42,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f31d0dced22947dcab547e82974d3557 2024-11-10T16:15:42,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/594e36b86f7947a69111829153a16306 2024-11-10T16:15:42,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/d90b69df58d7411d8a827bb20be7947a 2024-11-10T16:15:42,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/8e966095930947d6a980e519a5484494 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/8e966095930947d6a980e519a5484494 2024-11-10T16:15:42,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e6e8a7d2eee648289f0a728d8ddb7460 2024-11-10T16:15:42,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e110c06b56f8496eb39857784b2417f1 2024-11-10T16:15:42,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c42c8c0ae0b44156b31bb3c3cf011d40 2024-11-10T16:15:42,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5c730bc4ce784de7b89dae08ef866319 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5c730bc4ce784de7b89dae08ef866319 2024-11-10T16:15:42,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/381803c3048e4a8d8b206cca04a3b00c 2024-11-10T16:15:42,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/403267a06f7544c091f870d38a6c2395 2024-11-10T16:15:42,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/436e356b007240a8999fa96cb7078f1a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/436e356b007240a8999fa96cb7078f1a 2024-11-10T16:15:42,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/68f42b6c065c4eaa86ed329bbaa650ee 2024-11-10T16:15:42,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/f57849b1ab8d445a9af14bba9d66c45b 2024-11-10T16:15:42,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/3bc9974fc7a646c48927046293b88759 2024-11-10T16:15:42,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c758885e32b94dbf953ffc1307a04677 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c758885e32b94dbf953ffc1307a04677 2024-11-10T16:15:42,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c56ab939aab248619e8bf8b0d82b619e 2024-11-10T16:15:42,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/91c416f768814c4c91b2dd4e9fc9638b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/91c416f768814c4c91b2dd4e9fc9638b 2024-11-10T16:15:42,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/c871a32a398d4f7ea28710b6937729ef 2024-11-10T16:15:42,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/89f935236c7b49af9f7deb7318810381 2024-11-10T16:15:42,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/bedacef4bf5b40df8f86bf97fa947a94 2024-11-10T16:15:42,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/e8ccfebfd5714cd49cdfe39dd41da0f4 2024-11-10T16:15:42,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/7f0124add6e5448cbc0d8f6f5c3bd770 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/7f0124add6e5448cbc0d8f6f5c3bd770 2024-11-10T16:15:42,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/00ba6771cdfa4836ad345a592bd1dfd5 2024-11-10T16:15:42,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/5d2c3f2334b2400c871265ea20388260 2024-11-10T16:15:42,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/2ce3aa89e71944f29b15c06603c51239, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/617a2b66f93d4e4aa134cbc00fa0366b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/6df7531c981b46568539db39d7db7b39, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/acefa7cf67f143dfb28d6731cec4d5a1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7f29bd99930d48af98980f6eeccc40fc, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c753768136f84ee58f250824f71337d1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b] to archive 2024-11-10T16:15:42,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:42,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/cd0bc2f391bc4e6cabd19b269ce94ad0 2024-11-10T16:15:42,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d040610836a640009ffc62f99b73fb09 2024-11-10T16:15:42,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/2ce3aa89e71944f29b15c06603c51239 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/2ce3aa89e71944f29b15c06603c51239 2024-11-10T16:15:42,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5077aced6d4944e19555c630a9bfeb07 2024-11-10T16:15:42,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c655505c8b244c64b0be907f0ec549ef 2024-11-10T16:15:42,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5f0098a8f9574a19806fdc473b99baec 2024-11-10T16:15:42,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/617a2b66f93d4e4aa134cbc00fa0366b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/617a2b66f93d4e4aa134cbc00fa0366b 2024-11-10T16:15:42,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/f19f293a2a5c4fdfb716d060833ac15d 2024-11-10T16:15:42,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/498b635678a04673b97468b0e450ac52 2024-11-10T16:15:42,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/6df7531c981b46568539db39d7db7b39 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/6df7531c981b46568539db39d7db7b39 2024-11-10T16:15:42,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/622b5220926f4050aeb0cc91d0abe2db 2024-11-10T16:15:42,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/268f36b857764aef935a404670fe662c 2024-11-10T16:15:42,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/9fba5e6aa4fd4cc09752733d96bb0eb8 2024-11-10T16:15:42,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/acefa7cf67f143dfb28d6731cec4d5a1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/acefa7cf67f143dfb28d6731cec4d5a1 2024-11-10T16:15:42,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/357bf83e2e584a68829dce3e288d7149 2024-11-10T16:15:42,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d124d713f7764015973566de445c50af 2024-11-10T16:15:42,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7f29bd99930d48af98980f6eeccc40fc to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7f29bd99930d48af98980f6eeccc40fc 2024-11-10T16:15:42,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/7754bb2acd7c4004b019daaa327b5224 2024-11-10T16:15:42,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c3ae9595656349d8b4f45c9720ee9a9e 2024-11-10T16:15:42,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/723e55acbea043ebb275fa97e22802c9 2024-11-10T16:15:42,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c753768136f84ee58f250824f71337d1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/c753768136f84ee58f250824f71337d1 2024-11-10T16:15:42,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/db8ce9051b0b46cf9a857630b66af35a 2024-11-10T16:15:42,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/728b295f29934409ad5b188b7d3b735b 2024-11-10T16:15:42,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d4ec193f23e346a1a88a2405d009348b 2024-11-10T16:15:42,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/327110a6d0584630b1efb2eff668723e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/4cbe217d31f641eb9545d903b7021d07, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fd3fce0467af4e4eb68c5e163351ff99, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f9547ab0787f42f0bcf3d27ddd759c47, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/74f4ef7def6547d8a5860629e7676c52, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/14b2f4abcbfe4c3fbcb939e737904c3a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c] to archive 2024-11-10T16:15:42,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:15:42,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/567acbdfe0a5450da1fd09ceaf6a65b3 2024-11-10T16:15:42,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/5b358c1c31c44413bcc0b1c389382348 2024-11-10T16:15:42,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/327110a6d0584630b1efb2eff668723e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/327110a6d0584630b1efb2eff668723e 2024-11-10T16:15:42,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b0869b7ecc7642db8e59fc3366526165 2024-11-10T16:15:42,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fce05845483e465785bd62ffb26cb826 2024-11-10T16:15:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/913945e3cef949dfbbfdeb2f831ea159 2024-11-10T16:15:42,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/4cbe217d31f641eb9545d903b7021d07 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/4cbe217d31f641eb9545d903b7021d07 2024-11-10T16:15:42,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f5264bbae6134d00ac959359553d2b0b 2024-11-10T16:15:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f564f15dc96e4f1dacab687499f0af9b 2024-11-10T16:15:42,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fd3fce0467af4e4eb68c5e163351ff99 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/fd3fce0467af4e4eb68c5e163351ff99 2024-11-10T16:15:42,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/07ed72ce614a46a58bcc2459cc240521 2024-11-10T16:15:42,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/9a6a3c00c4194de2aca6ea032a52f124 2024-11-10T16:15:42,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/303b9842b7c54363b7ab11a21b880dc9 2024-11-10T16:15:42,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f9547ab0787f42f0bcf3d27ddd759c47 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/f9547ab0787f42f0bcf3d27ddd759c47 2024-11-10T16:15:42,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/c31c4dbd1d884834af124c9e53fa08e3 2024-11-10T16:15:42,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e3e92c43153742bb9905516392b76b35 2024-11-10T16:15:42,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/74f4ef7def6547d8a5860629e7676c52 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/74f4ef7def6547d8a5860629e7676c52 2024-11-10T16:15:42,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/b58ddfa39b2b47d0b56de67fcaa2ae30 2024-11-10T16:15:42,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/e83aa8ccc1824b5a8d6a7409c7f4c323 2024-11-10T16:15:42,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/57454234f3a44352bdee5ea809396ba0 2024-11-10T16:15:42,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/14b2f4abcbfe4c3fbcb939e737904c3a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/14b2f4abcbfe4c3fbcb939e737904c3a 2024-11-10T16:15:42,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/165bfd4a5f6443ceb51fe693fdb54b74 2024-11-10T16:15:42,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/7fff2ad3b6674cc490627b7d3260ed81 2024-11-10T16:15:42,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/da833c9015f249fdb31a93ba317bfd4c 2024-11-10T16:15:42,319 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/recovered.edits/449.seqid, newMaxSeqId=449, maxSeqId=1 2024-11-10T16:15:42,319 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7. 2024-11-10T16:15:42,319 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 76c86cef11e7fc54de9404956c98cfb7: Waiting for close lock at 1731255342255Running coprocessor pre-close hooks at 1731255342255Disabling compacts and flushes for region at 1731255342255Disabling writes for close at 1731255342255Writing region close event to WAL at 1731255342316 (+61 ms)Running coprocessor post-close hooks at 1731255342319 (+3 ms)Closed at 1731255342319 2024-11-10T16:15:42,321 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:42,321 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=76c86cef11e7fc54de9404956c98cfb7, regionState=CLOSED 2024-11-10T16:15:42,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:42,324 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T16:15:42,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 76c86cef11e7fc54de9404956c98cfb7, server=8e3de9160f9f,44105,1731255200149 in 220 msec 2024-11-10T16:15:42,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-10T16:15:42,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=76c86cef11e7fc54de9404956c98cfb7, UNASSIGN in 225 msec 2024-11-10T16:15:42,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-10T16:15:42,328 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 227 msec 2024-11-10T16:15:42,329 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255342329"}]},"ts":"1731255342329"} 2024-11-10T16:15:42,330 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:15:42,330 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:15:42,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 236 msec 2024-11-10T16:15:42,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T16:15:42,414 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:42,414 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:15:42,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T16:15:42,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,419 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:42,420 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/recovered.edits] 2024-11-10T16:15:42,423 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/aaaa08393e214c308bfffd4ee43f5b20 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/aaaa08393e214c308bfffd4ee43f5b20 2024-11-10T16:15:42,423 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/dd626579e08d4a36bfbd8294c604e82a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/dd626579e08d4a36bfbd8294c604e82a 2024-11-10T16:15:42,424 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/eb68fa1977264044928d9998bf5eb422 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/A/eb68fa1977264044928d9998bf5eb422 2024-11-10T16:15:42,426 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/3aff85f157224520b90189274fe74c5d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/3aff85f157224520b90189274fe74c5d 2024-11-10T16:15:42,427 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5b88800d445e475c83a18d5dae8bef5e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/5b88800d445e475c83a18d5dae8bef5e 2024-11-10T16:15:42,428 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d9317deef2504e0eb579bd8c45832405 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/B/d9317deef2504e0eb579bd8c45832405 2024-11-10T16:15:42,429 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/092f1d65ad61495b83685d9f05fdf851 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/092f1d65ad61495b83685d9f05fdf851 2024-11-10T16:15:42,430 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/09f149390f6c4fbe8e2290a2f74b49c3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/09f149390f6c4fbe8e2290a2f74b49c3 2024-11-10T16:15:42,431 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/6aab20ca3cda452f965b3c5574f95454 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/C/6aab20ca3cda452f965b3c5574f95454 2024-11-10T16:15:42,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/recovered.edits/449.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7/recovered.edits/449.seqid 2024-11-10T16:15:42,433 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/76c86cef11e7fc54de9404956c98cfb7 2024-11-10T16:15:42,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:15:42,435 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,437 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:15:42,438 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:15:42,439 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,439 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:15:42,439 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255342439"}]},"ts":"9223372036854775807"} 2024-11-10T16:15:42,441 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:15:42,441 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 76c86cef11e7fc54de9404956c98cfb7, NAME => 'TestAcidGuarantees,,1731255316632.76c86cef11e7fc54de9404956c98cfb7.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:15:42,441 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:15:42,441 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255342441"}]},"ts":"9223372036854775807"} 2024-11-10T16:15:42,442 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:15:42,443 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-10T16:15:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T16:15:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:15:42,524 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:42,534 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 250), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=313 (was 304) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7018 (was 7039) 2024-11-10T16:15:42,543 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=7018 2024-11-10T16:15:42,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:15:42,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:15:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:42,546 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T16:15:42,546 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:42,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-10T16:15:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:42,547 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T16:15:42,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742323_1499 (size=1061) 2024-11-10T16:15:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:42,954 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255 2024-11-10T16:15:42,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742324_1500 (size=53) 2024-11-10T16:15:43,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 86e0154132bb6973fbfbd611efdf9b15, disabling compactions & flushes 2024-11-10T16:15:43,360 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. after waiting 0 ms 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,360 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,360 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 86e0154132bb6973fbfbd611efdf9b15: Waiting for close lock at 1731255343360Disabling compacts and flushes for region at 1731255343360Disabling writes for close at 1731255343360Writing region close event to WAL at 1731255343360Closed at 1731255343360 2024-11-10T16:15:43,361 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T16:15:43,361 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731255343361"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731255343361"}]},"ts":"1731255343361"} 2024-11-10T16:15:43,362 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T16:15:43,363 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T16:15:43,363 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255343363"}]},"ts":"1731255343363"} 2024-11-10T16:15:43,364 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T16:15:43,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, ASSIGN}] 2024-11-10T16:15:43,365 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, ASSIGN 2024-11-10T16:15:43,366 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, ASSIGN; state=OFFLINE, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=false 2024-11-10T16:15:43,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:43,518 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, ASSIGN because future has completed 2024-11-10T16:15:43,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:43,672 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,673 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:15:43,673 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,673 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:15:43,673 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,673 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,674 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,675 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:43,675 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName A 2024-11-10T16:15:43,675 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:43,676 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:43,676 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,677 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:43,677 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName B 2024-11-10T16:15:43,677 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:43,677 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:43,677 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,678 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:43,678 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName C 2024-11-10T16:15:43,678 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:43,678 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:43,678 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,678 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,679 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,680 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,680 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,680 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:15:43,681 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:43,683 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T16:15:43,683 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened 86e0154132bb6973fbfbd611efdf9b15; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71487249, jitterRate=0.06524302065372467}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:15:43,683 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:43,684 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for 86e0154132bb6973fbfbd611efdf9b15: Running coprocessor pre-open hook at 1731255343673Writing region info on filesystem at 1731255343673Initializing all the Stores at 1731255343674 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255343674Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255343674Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255343674Cleaning up temporary data from old regions at 1731255343680 (+6 ms)Running coprocessor post-open hooks at 1731255343683 (+3 ms)Region opened successfully at 1731255343684 (+1 ms) 2024-11-10T16:15:43,685 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., pid=140, masterSystemTime=1731255343670 2024-11-10T16:15:43,687 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,687 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:43,687 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=OPEN, openSeqNum=2, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:43,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:43,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T16:15:43,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 in 171 msec 2024-11-10T16:15:43,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-10T16:15:43,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, ASSIGN in 326 msec 2024-11-10T16:15:43,693 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T16:15:43,693 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255343693"}]},"ts":"1731255343693"} 2024-11-10T16:15:43,694 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T16:15:43,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T16:15:43,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-10T16:15:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-10T16:15:44,693 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:44,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:44,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:44,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:44,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:44,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3e560f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:44,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:44,697 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:44,697 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5885e501, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:44,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:44,698 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:44,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:44,700 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T16:15:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8e3de9160f9f,41631,1731255199363 2024-11-10T16:15:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@663a465a 2024-11-10T16:15:44,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T16:15:44,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T16:15:44,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T16:15:44,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T16:15:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T16:15:44,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742325_1501 (size=1097) 2024-11-10T16:15:45,113 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T16:15:45,113 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T16:15:45,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:15:45,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE}] 2024-11-10T16:15:45,118 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE 2024-11-10T16:15:45,119 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,120 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE because future has completed 2024-11-10T16:15:45,120 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:15:45,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:45,272 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,273 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:15:45,273 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 86e0154132bb6973fbfbd611efdf9b15, disabling compactions & flushes 2024-11-10T16:15:45,273 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,273 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,273 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. after waiting 0 ms 2024-11-10T16:15:45,273 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,276 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T16:15:45,276 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,276 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 86e0154132bb6973fbfbd611efdf9b15: Waiting for close lock at 1731255345273Running coprocessor pre-close hooks at 1731255345273Disabling compacts and flushes for region at 1731255345273Disabling writes for close at 1731255345273Writing region close event to WAL at 1731255345273Running coprocessor post-close hooks at 1731255345276 (+3 ms)Closed at 1731255345276 2024-11-10T16:15:45,276 WARN [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: 86e0154132bb6973fbfbd611efdf9b15 to self. 2024-11-10T16:15:45,278 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,278 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=CLOSED 2024-11-10T16:15:45,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:45,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-10T16:15:45,282 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 in 160 msec 2024-11-10T16:15:45,282 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE; state=CLOSED, location=8e3de9160f9f,44105,1731255200149; forceNewPlan=false, retain=true 2024-11-10T16:15:45,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=OPENING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE because future has completed 2024-11-10T16:15:45,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:15:45,589 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,589 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} 2024-11-10T16:15:45,589 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,589 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T16:15:45,589 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,589 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,591 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,591 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:45,591 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName A 2024-11-10T16:15:45,592 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:45,593 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:45,593 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,594 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:45,594 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName B 2024-11-10T16:15:45,594 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:45,594 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:45,594 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,595 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T16:15:45,595 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 86e0154132bb6973fbfbd611efdf9b15 columnFamilyName C 2024-11-10T16:15:45,595 DEBUG [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:45,595 INFO [StoreOpener-86e0154132bb6973fbfbd611efdf9b15-1 {}] regionserver.HStore(327): Store=86e0154132bb6973fbfbd611efdf9b15/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T16:15:45,595 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,595 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,596 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,596 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,597 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,597 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,598 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T16:15:45,598 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,599 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened 86e0154132bb6973fbfbd611efdf9b15; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59520294, jitterRate=-0.11307850480079651}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T16:15:45,599 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,600 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for 86e0154132bb6973fbfbd611efdf9b15: Running coprocessor pre-open hook at 1731255345590Writing region info on filesystem at 1731255345590Initializing all the Stores at 1731255345590Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255345590Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255345590Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731255345590Cleaning up temporary data from old regions at 1731255345597 (+7 ms)Running coprocessor post-open hooks at 1731255345599 (+2 ms)Region opened successfully at 1731255345600 (+1 ms) 2024-11-10T16:15:45,600 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., pid=145, masterSystemTime=1731255345586 2024-11-10T16:15:45,602 DEBUG [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,602 INFO [RS_OPEN_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,602 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=OPEN, openSeqNum=5, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:15:45,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-10T16:15:45,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 in 169 msec 2024-11-10T16:15:45,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T16:15:45,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, REOPEN/MOVE in 488 msec 2024-11-10T16:15:45,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-10T16:15:45,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-10T16:15:45,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T16:15:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-10T16:15:45,612 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:45,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3261e003, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,615 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2855bba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,617 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d9fe4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b7d7b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,619 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa24f42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c3f296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,622 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,622 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a02416d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d36138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a604e60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,626 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,626 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,627 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d298a71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36ea48b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,629 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,629 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7c910d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,630 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,631 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573a96c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,631 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,632 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b729619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,634 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b859563, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,635 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19ff54d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,636 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab79160, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dd230ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8e3de9160f9f,41631,-1 for getting cluster id 2024-11-10T16:15:45,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T16:15:45,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e8791941-1e0e-47f0-b506-1348b132a954' 2024-11-10T16:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T16:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e8791941-1e0e-47f0-b506-1348b132a954" 2024-11-10T16:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa8c440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8e3de9160f9f,41631,-1] 2024-11-10T16:15:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T16:15:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:15:45,640 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T16:15:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a46f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T16:15:45,641 DEBUG [Thread-2261 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,641 DEBUG [Thread-2254 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,641 DEBUG [Thread-2265 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,642 DEBUG [Thread-2252 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,642 DEBUG [Thread-2248 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,642 DEBUG [Thread-2256 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:45,643 DEBUG [Thread-2263 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,643 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,643 DEBUG [Thread-2250 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,644 DEBUG [Thread-2259 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,645 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8e3de9160f9f,44105,1731255200149, seqNum=-1] 2024-11-10T16:15:45,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T16:15:45,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T16:15:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-10T16:15:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:45,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:45,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,653 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5] 2024-11-10T16:15:45,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:45,656 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:45,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:45,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:45,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107e6befbadb464feb830e09fb78a04d34_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255345655/Put/seqid=0 2024-11-10T16:15:45,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742326_1502 (size=12154) 2024-11-10T16:15:45,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255405703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255405705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255405705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,711 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255345702, max timestamp=1731255345708Num uniques 859; 2024-11-10T16:15:45,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:45,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255345702, max timestamp=1731255345708Num uniques 859; 2024-11-10T16:15:45,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:45,711 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255345702, max timestamp=1731255345708Num uniques 859; 2024-11-10T16:15:45,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255405771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255405771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:45,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255405813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255405813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255405813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255405883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255405883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:45,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:45,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:45,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:45,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255406023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255406023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255406023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,082 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:46,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107e6befbadb464feb830e09fb78a04d34_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e6befbadb464feb830e09fb78a04d34_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:46,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b8910e3d618047e1b2c390146a862ed3, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:46,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b8910e3d618047e1b2c390146a862ed3 is 175, key is test_row_0/A:col10/1731255345655/Put/seqid=0 2024-11-10T16:15:46,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742327_1503 (size=30955) 2024-11-10T16:15:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255406093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255406093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:46,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255406333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255406334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255406334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255406403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255406403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,490 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b8910e3d618047e1b2c390146a862ed3 2024-11-10T16:15:46,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/fe49ec9f71e6490a8bca08a5894cc7fb is 50, key is test_row_0/B:col10/1731255345655/Put/seqid=0 2024-11-10T16:15:46,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742328_1504 (size=12001) 2024-11-10T16:15:46,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255406843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255406843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255406843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:46,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:46,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/fe49ec9f71e6490a8bca08a5894cc7fb 2024-11-10T16:15:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255406923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255406923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:46,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:46,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/05ef826758bc44848a9d13a90a1c8390 is 50, key is test_row_0/C:col10/1731255345655/Put/seqid=0 2024-11-10T16:15:46,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742329_1505 (size=12001) 2024-11-10T16:15:47,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/05ef826758bc44848a9d13a90a1c8390 2024-11-10T16:15:47,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b8910e3d618047e1b2c390146a862ed3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3 2024-11-10T16:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:47,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3, entries=150, sequenceid=15, filesize=30.2 K 2024-11-10T16:15:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/fe49ec9f71e6490a8bca08a5894cc7fb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb 2024-11-10T16:15:47,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T16:15:47,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/05ef826758bc44848a9d13a90a1c8390 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390 2024-11-10T16:15:47,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T16:15:47,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 86e0154132bb6973fbfbd611efdf9b15 in 1697ms, sequenceid=15, compaction requested=false 2024-11-10T16:15:47,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T16:15:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:47,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:47,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044fc9eab9ffa41078c0a8ea91e75f5ba_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255345769/Put/seqid=0 2024-11-10T16:15:47,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742330_1506 (size=12154) 2024-11-10T16:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:47,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T16:15:47,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:47,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:47,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255407869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255407874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255407875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:47,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044fc9eab9ffa41078c0a8ea91e75f5ba_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fc9eab9ffa41078c0a8ea91e75f5ba_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:47,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a39f7c79d5264c2084ab1954d0edcd08, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:47,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a39f7c79d5264c2084ab1954d0edcd08 is 175, key is test_row_0/A:col10/1731255345769/Put/seqid=0 2024-11-10T16:15:47,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742331_1507 (size=30955) 2024-11-10T16:15:47,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255407933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,938 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255407943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,948 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255407983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255407984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255407984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:47,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255408193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255408194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255408194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,314 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a39f7c79d5264c2084ab1954d0edcd08 2024-11-10T16:15:48,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/9b8d66799e1b49109eff1bff6aecc96a is 50, key is test_row_0/B:col10/1731255345769/Put/seqid=0 2024-11-10T16:15:48,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742332_1508 (size=12001) 2024-11-10T16:15:48,367 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T16:15:48,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255408503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255408503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255408503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:48,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:48,726 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/9b8d66799e1b49109eff1bff6aecc96a 2024-11-10T16:15:48,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4f7d681a24dd4045be35a2cad2908cae is 50, key is test_row_0/C:col10/1731255345769/Put/seqid=0 2024-11-10T16:15:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742333_1509 (size=12001) 2024-11-10T16:15:48,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4f7d681a24dd4045be35a2cad2908cae 2024-11-10T16:15:48,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a39f7c79d5264c2084ab1954d0edcd08 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08 2024-11-10T16:15:48,745 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08, entries=150, sequenceid=51, filesize=30.2 K 2024-11-10T16:15:48,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/9b8d66799e1b49109eff1bff6aecc96a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a 2024-11-10T16:15:48,749 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T16:15:48,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4f7d681a24dd4045be35a2cad2908cae as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae 2024-11-10T16:15:48,753 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T16:15:48,754 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 86e0154132bb6973fbfbd611efdf9b15 in 1261ms, sequenceid=51, compaction requested=false 2024-11-10T16:15:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:48,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-10T16:15:48,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-10T16:15:48,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T16:15:48,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0980 sec 2024-11-10T16:15:48,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.1150 sec 2024-11-10T16:15:49,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:49,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:49,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:49,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111053dcd3c0557b40db87b498f1e7ebf9e0_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:49,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742334_1510 (size=14594) 2024-11-10T16:15:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255409044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255409045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255409046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255409153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255409153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255409153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255409363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255409363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255409363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:49,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111053dcd3c0557b40db87b498f1e7ebf9e0_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053dcd3c0557b40db87b498f1e7ebf9e0_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:49,436 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1e178ece65fb4f90afaefd30c818c519, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:49,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1e178ece65fb4f90afaefd30c818c519 is 175, key is test_row_0/A:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:49,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742335_1511 (size=39549) 2024-11-10T16:15:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255409673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255409674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255409674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T16:15:49,803 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:49,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-10T16:15:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:49,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:49,806 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:49,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:49,841 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1e178ece65fb4f90afaefd30c818c519 2024-11-10T16:15:49,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/eda0db07e7974d41912c4b623a01b401 is 50, key is test_row_0/B:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:49,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742336_1512 (size=12001) 2024-11-10T16:15:49,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/eda0db07e7974d41912c4b623a01b401 2024-11-10T16:15:49,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a25fcbaf6f0045ce8cf06cb5942375e8 is 50, key is test_row_0/C:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742337_1513 (size=12001) 2024-11-10T16:15:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:49,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255409953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,958 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:49,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T16:15:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:49,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:49,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255409963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:49,967 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:49,967 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:50,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T16:15:50,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:50,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255410183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:50,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255410183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:50,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255410184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:50,189 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:50,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T16:15:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:50,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a25fcbaf6f0045ce8cf06cb5942375e8 2024-11-10T16:15:50,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1e178ece65fb4f90afaefd30c818c519 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519 2024-11-10T16:15:50,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519, entries=200, sequenceid=64, filesize=38.6 K 2024-11-10T16:15:50,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/eda0db07e7974d41912c4b623a01b401 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401 2024-11-10T16:15:50,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401, entries=150, sequenceid=64, filesize=11.7 K 2024-11-10T16:15:50,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a25fcbaf6f0045ce8cf06cb5942375e8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8 2024-11-10T16:15:50,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8, entries=150, sequenceid=64, filesize=11.7 K 2024-11-10T16:15:50,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 86e0154132bb6973fbfbd611efdf9b15 in 1270ms, sequenceid=64, compaction requested=true 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:50,286 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:50,286 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:50,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:50,287 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:50,287 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:50,287 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:15:50,287 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:15:50,287 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,287 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,287 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.2 K 2024-11-10T16:15:50,287 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=99.1 K 2024-11-10T16:15:50,287 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,287 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519] 2024-11-10T16:15:50,288 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting fe49ec9f71e6490a8bca08a5894cc7fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731255345652 2024-11-10T16:15:50,288 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8910e3d618047e1b2c390146a862ed3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731255345652 2024-11-10T16:15:50,288 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8d66799e1b49109eff1bff6aecc96a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731255345761 2024-11-10T16:15:50,288 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39f7c79d5264c2084ab1954d0edcd08, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731255345761 2024-11-10T16:15:50,288 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting eda0db07e7974d41912c4b623a01b401, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:50,289 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e178ece65fb4f90afaefd30c818c519, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:50,296 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#434 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:50,297 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/53eccdb467534eb0bc207deb41528b13 is 50, key is test_row_0/B:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:50,299 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:50,301 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102088619647b44cc9a2ac8457b5a26e6c_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:50,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742338_1514 (size=12104) 2024-11-10T16:15:50,303 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102088619647b44cc9a2ac8457b5a26e6c_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:50,303 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102088619647b44cc9a2ac8457b5a26e6c_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:50,310 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/53eccdb467534eb0bc207deb41528b13 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/53eccdb467534eb0bc207deb41528b13 2024-11-10T16:15:50,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742339_1515 (size=4469) 2024-11-10T16:15:50,316 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#435 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:50,317 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b7bfa2142c1b4d8a927565a4c6e95c69 is 175, key is test_row_0/A:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:50,322 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 53eccdb467534eb0bc207deb41528b13(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:50,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:50,322 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255350286; duration=0sec 2024-11-10T16:15:50,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:50,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:15:50,322 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:50,324 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:50,324 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:15:50,324 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,324 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.2 K 2024-11-10T16:15:50,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742340_1516 (size=31058) 2024-11-10T16:15:50,325 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ef826758bc44848a9d13a90a1c8390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731255345652 2024-11-10T16:15:50,325 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f7d681a24dd4045be35a2cad2908cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731255345761 2024-11-10T16:15:50,326 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a25fcbaf6f0045ce8cf06cb5942375e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:50,337 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:50,337 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/62a0667f7d46432ebdf2129b68343633 is 50, key is test_row_0/C:col10/1731255347868/Put/seqid=0 2024-11-10T16:15:50,339 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b7bfa2142c1b4d8a927565a4c6e95c69 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69 2024-11-10T16:15:50,344 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into b7bfa2142c1b4d8a927565a4c6e95c69(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:50,344 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:50,344 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255350286; duration=0sec 2024-11-10T16:15:50,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742341_1517 (size=12104) 2024-11-10T16:15:50,344 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:50,344 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:15:50,349 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/62a0667f7d46432ebdf2129b68343633 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/62a0667f7d46432ebdf2129b68343633 2024-11-10T16:15:50,356 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into 62a0667f7d46432ebdf2129b68343633(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:50,356 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:50,356 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255350286; duration=0sec 2024-11-10T16:15:50,356 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:50,356 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:15:50,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:50,418 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:50,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd5053747ae04ce9ba66a75584f8a6dc_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255349042/Put/seqid=0 2024-11-10T16:15:50,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742342_1518 (size=12154) 2024-11-10T16:15:50,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:50,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:50,835 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd5053747ae04ce9ba66a75584f8a6dc_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd5053747ae04ce9ba66a75584f8a6dc_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:50,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ad9b8dd29a2c4c96a26296812bfa3dce, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ad9b8dd29a2c4c96a26296812bfa3dce is 175, key is test_row_0/A:col10/1731255349042/Put/seqid=0 2024-11-10T16:15:50,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742343_1519 (size=30955) 2024-11-10T16:15:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:51,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255411215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255411216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255411220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,240 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ad9b8dd29a2c4c96a26296812bfa3dce 2024-11-10T16:15:51,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5bd35bfe7f754fc69f64599dcd0cf909 is 50, key is test_row_0/B:col10/1731255349042/Put/seqid=0 2024-11-10T16:15:51,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742344_1520 (size=12001) 2024-11-10T16:15:51,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255411323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255411324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255411333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255411533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255411533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255411543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,651 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5bd35bfe7f754fc69f64599dcd0cf909 2024-11-10T16:15:51,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/32757b09c0544820b63d69b638c721b1 is 50, key is test_row_0/C:col10/1731255349042/Put/seqid=0 2024-11-10T16:15:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742345_1521 (size=12001) 2024-11-10T16:15:51,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255411843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255411843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255411853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:51,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:51,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:52,061 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/32757b09c0544820b63d69b638c721b1 2024-11-10T16:15:52,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ad9b8dd29a2c4c96a26296812bfa3dce as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce 2024-11-10T16:15:52,069 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce, entries=150, sequenceid=86, filesize=30.2 K 2024-11-10T16:15:52,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5bd35bfe7f754fc69f64599dcd0cf909 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909 2024-11-10T16:15:52,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T16:15:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/32757b09c0544820b63d69b638c721b1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1 2024-11-10T16:15:52,076 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1, entries=150, sequenceid=86, filesize=11.7 K 2024-11-10T16:15:52,078 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 86e0154132bb6973fbfbd611efdf9b15 in 1660ms, sequenceid=86, compaction requested=false 2024-11-10T16:15:52,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:52,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:52,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-10T16:15:52,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-10T16:15:52,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T16:15:52,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2730 sec 2024-11-10T16:15:52,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.2780 sec 2024-11-10T16:15:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:52,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:15:52,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:52,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:52,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:52,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:52,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:52,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:52,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107e9bdf513cdf4c38afa933f53d52e9bc_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:52,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742346_1522 (size=14594) 2024-11-10T16:15:52,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107e9bdf513cdf4c38afa933f53d52e9bc_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e9bdf513cdf4c38afa933f53d52e9bc_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:52,379 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c9019302d1b64a5aaeda5e414f83cace, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:52,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c9019302d1b64a5aaeda5e414f83cace is 175, key is test_row_0/A:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:52,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742347_1523 (size=39549) 2024-11-10T16:15:52,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255412390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255412391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255412392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255412503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255412503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255412503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255412713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255412713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255412714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:52,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:52,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c9019302d1b64a5aaeda5e414f83cace 2024-11-10T16:15:52,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/cd1b96b02d37462ca706c75ece96bc0d is 50, key is test_row_0/B:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:52,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742348_1524 (size=12001) 2024-11-10T16:15:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255413023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255413023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255413023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,026 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,027 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/cd1b96b02d37462ca706c75ece96bc0d 2024-11-10T16:15:53,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/92e79696f8fa45df8dc1ce7c60ddcb6b is 50, key is test_row_0/C:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:53,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742349_1525 (size=12001) 2024-11-10T16:15:53,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255413534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255413534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255413534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,538 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:53,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:53,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/92e79696f8fa45df8dc1ce7c60ddcb6b 2024-11-10T16:15:53,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c9019302d1b64a5aaeda5e414f83cace as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace 2024-11-10T16:15:53,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace, entries=200, sequenceid=100, filesize=38.6 K 2024-11-10T16:15:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/cd1b96b02d37462ca706c75ece96bc0d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d 2024-11-10T16:15:53,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T16:15:53,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/92e79696f8fa45df8dc1ce7c60ddcb6b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b 2024-11-10T16:15:53,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T16:15:53,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 86e0154132bb6973fbfbd611efdf9b15 in 1266ms, sequenceid=100, compaction requested=true 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:53,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:53,622 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:53,622 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:53,623 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:53,623 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:53,623 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:15:53,623 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:15:53,623 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:53,623 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:53,623 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=99.2 K 2024-11-10T16:15:53,623 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/53eccdb467534eb0bc207deb41528b13, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.3 K 2024-11-10T16:15:53,624 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:53,624 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace] 2024-11-10T16:15:53,624 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 53eccdb467534eb0bc207deb41528b13, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:53,624 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7bfa2142c1b4d8a927565a4c6e95c69, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:53,624 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd35bfe7f754fc69f64599dcd0cf909, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255349038 2024-11-10T16:15:53,624 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9b8dd29a2c4c96a26296812bfa3dce, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255349038 2024-11-10T16:15:53,625 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9019302d1b64a5aaeda5e414f83cace, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:53,625 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1b96b02d37462ca706c75ece96bc0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:53,632 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#443 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:53,633 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/d19800e45ee34b02a450dda53f903932 is 50, key is test_row_0/B:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:53,633 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:53,635 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109c762964c9484c1db87d56bb1a9b72e3_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:53,637 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109c762964c9484c1db87d56bb1a9b72e3_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:53,638 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109c762964c9484c1db87d56bb1a9b72e3_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:53,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742350_1526 (size=12207) 2024-11-10T16:15:53,649 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/d19800e45ee34b02a450dda53f903932 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/d19800e45ee34b02a450dda53f903932 2024-11-10T16:15:53,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742351_1527 (size=4469) 2024-11-10T16:15:53,651 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#444 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:53,651 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ef39c64a4be945a283ec10371a0ca555 is 175, key is test_row_0/A:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:53,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742352_1528 (size=31161) 2024-11-10T16:15:53,656 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into d19800e45ee34b02a450dda53f903932(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:53,656 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:53,656 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255353622; duration=0sec 2024-11-10T16:15:53,656 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:53,656 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:15:53,656 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:15:53,657 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:15:53,657 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:15:53,657 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:53,657 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/62a0667f7d46432ebdf2129b68343633, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.3 K 2024-11-10T16:15:53,658 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 62a0667f7d46432ebdf2129b68343633, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731255347861 2024-11-10T16:15:53,658 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 32757b09c0544820b63d69b638c721b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731255349038 2024-11-10T16:15:53,659 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e79696f8fa45df8dc1ce7c60ddcb6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:53,660 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ef39c64a4be945a283ec10371a0ca555 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555 2024-11-10T16:15:53,666 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:53,666 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7f13e43438cc414f98871ac8be8e8b9b is 50, key is test_row_0/C:col10/1731255352354/Put/seqid=0 2024-11-10T16:15:53,668 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into ef39c64a4be945a283ec10371a0ca555(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:53,668 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:53,668 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255353622; duration=0sec 2024-11-10T16:15:53,668 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:53,668 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:15:53,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742353_1529 (size=12207) 2024-11-10T16:15:53,681 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7f13e43438cc414f98871ac8be8e8b9b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f13e43438cc414f98871ac8be8e8b9b 2024-11-10T16:15:53,686 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into 7f13e43438cc414f98871ac8be8e8b9b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:53,686 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:53,686 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255353622; duration=0sec 2024-11-10T16:15:53,686 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:53,686 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:15:53,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T16:15:53,963 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:53,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-10T16:15:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:53,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:53,966 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:53,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:53,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:53,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:53,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:53,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111053b1ec5b2b004cf7af813aedc529e254_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255352389/Put/seqid=0 2024-11-10T16:15:53,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742354_1530 (size=12154) 2024-11-10T16:15:54,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255414007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255414008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:54,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255414113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,117 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255414123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:54,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255414323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255414333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,397 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:54,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111053b1ec5b2b004cf7af813aedc529e254_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053b1ec5b2b004cf7af813aedc529e254_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:54,401 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/fb99afb6593342bcacedc7c4a373526a, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:54,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/fb99afb6593342bcacedc7c4a373526a is 175, key is test_row_0/A:col10/1731255352389/Put/seqid=0 2024-11-10T16:15:54,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742355_1531 (size=30955) 2024-11-10T16:15:54,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255414543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255414543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,545 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,545 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255414553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,556 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:54,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255414633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255414643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:54,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/fb99afb6593342bcacedc7c4a373526a 2024-11-10T16:15:54,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/aee70aa163fc4ea4ac64779a7ddeda08 is 50, key is test_row_0/B:col10/1731255352389/Put/seqid=0 2024-11-10T16:15:54,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742356_1532 (size=12001) 2024-11-10T16:15:54,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:54,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:54,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:54,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:54,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:55,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:55,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255415143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:55,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:55,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255415153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:55,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:55,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/aee70aa163fc4ea4ac64779a7ddeda08 2024-11-10T16:15:55,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4b06537789a3409498cc069b45cfa1fd is 50, key is test_row_0/C:col10/1731255352389/Put/seqid=0 2024-11-10T16:15:55,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742357_1533 (size=12001) 2024-11-10T16:15:55,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:55,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:55,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:55,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4b06537789a3409498cc069b45cfa1fd 2024-11-10T16:15:55,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/fb99afb6593342bcacedc7c4a373526a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a 2024-11-10T16:15:55,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a, entries=150, sequenceid=123, filesize=30.2 K 2024-11-10T16:15:55,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/aee70aa163fc4ea4ac64779a7ddeda08 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08 2024-11-10T16:15:55,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T16:15:55,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/4b06537789a3409498cc069b45cfa1fd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd 2024-11-10T16:15:55,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd, entries=150, sequenceid=123, filesize=11.7 K 2024-11-10T16:15:55,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 86e0154132bb6973fbfbd611efdf9b15 in 1657ms, sequenceid=123, compaction requested=false 2024-11-10T16:15:55,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:55,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:55,649 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:55,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044ce297b764144fdaa6f844adbd06aff_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255354007/Put/seqid=0 2024-11-10T16:15:55,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742358_1534 (size=12254) 2024-11-10T16:15:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:56,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111044ce297b764144fdaa6f844adbd06aff_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044ce297b764144fdaa6f844adbd06aff_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9036ab1cb6364cd3bac3f6653bf6697b, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:56,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9036ab1cb6364cd3bac3f6653bf6697b is 175, key is test_row_0/A:col10/1731255354007/Put/seqid=0 2024-11-10T16:15:56,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742359_1535 (size=31055) 2024-11-10T16:15:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:56,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:56,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:56,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255416199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255416203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255416313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255416313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,474 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9036ab1cb6364cd3bac3f6653bf6697b 2024-11-10T16:15:56,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/71bdbaabed454d13a2c3bcc1bae284ea is 50, key is test_row_0/B:col10/1731255354007/Put/seqid=0 2024-11-10T16:15:56,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742360_1536 (size=12101) 2024-11-10T16:15:56,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255416523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255416523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255416554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,557 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255416563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,566 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255416583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,586 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:15:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255416833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255416833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:56,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/71bdbaabed454d13a2c3bcc1bae284ea 2024-11-10T16:15:56,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a823b8d0688742f39c770cef390ddfa1 is 50, key is test_row_0/C:col10/1731255354007/Put/seqid=0 2024-11-10T16:15:56,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742361_1537 (size=12101) 2024-11-10T16:15:56,901 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a823b8d0688742f39c770cef390ddfa1 2024-11-10T16:15:56,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9036ab1cb6364cd3bac3f6653bf6697b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b 2024-11-10T16:15:56,908 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b, entries=150, sequenceid=135, filesize=30.3 K 2024-11-10T16:15:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/71bdbaabed454d13a2c3bcc1bae284ea as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea 2024-11-10T16:15:56,912 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea, entries=150, sequenceid=135, filesize=11.8 K 2024-11-10T16:15:56,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/a823b8d0688742f39c770cef390ddfa1 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1 2024-11-10T16:15:56,916 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1, entries=150, sequenceid=135, filesize=11.8 K 2024-11-10T16:15:56,916 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 86e0154132bb6973fbfbd611efdf9b15 in 1267ms, sequenceid=135, compaction requested=true 2024-11-10T16:15:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-10T16:15:56,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-10T16:15:56,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T16:15:56,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9520 sec 2024-11-10T16:15:56,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.9560 sec 2024-11-10T16:15:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:57,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:57,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:15:57,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111048493e4c6b4b4c2f8eee2ea02ac919e8_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:57,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742362_1538 (size=14794) 2024-11-10T16:15:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255417401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255417401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255417513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255417513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255417723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255417723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:57,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:57,764 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:57,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111048493e4c6b4b4c2f8eee2ea02ac919e8_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048493e4c6b4b4c2f8eee2ea02ac919e8_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:57,769 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a3b81c098aee4b93bd9105b6b9633a4a, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:57,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a3b81c098aee4b93bd9105b6b9633a4a is 175, key is test_row_0/A:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:57,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742363_1539 (size=39749) 2024-11-10T16:15:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255418033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255418033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T16:15:58,123 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:58,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-10T16:15:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T16:15:58,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:58,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:58,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:58,173 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a3b81c098aee4b93bd9105b6b9633a4a 2024-11-10T16:15:58,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/b991b23b91794427a7ccbd0427df863b is 50, key is test_row_0/B:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:58,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742364_1540 (size=12151) 2024-11-10T16:15:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T16:15:58,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T16:15:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T16:15:58,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:58,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255418543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255418544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:58,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/b991b23b91794427a7ccbd0427df863b 2024-11-10T16:15:58,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T16:15:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:58,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/64f5f2ca3f604d36bbe00445c8e93286 is 50, key is test_row_0/C:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:58,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742365_1541 (size=12151) 2024-11-10T16:15:58,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/64f5f2ca3f604d36bbe00445c8e93286 2024-11-10T16:15:58,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/a3b81c098aee4b93bd9105b6b9633a4a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a 2024-11-10T16:15:58,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a, entries=200, sequenceid=156, filesize=38.8 K 2024-11-10T16:15:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/b991b23b91794427a7ccbd0427df863b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b 2024-11-10T16:15:58,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T16:15:58,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/64f5f2ca3f604d36bbe00445c8e93286 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286 2024-11-10T16:15:58,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T16:15:58,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 86e0154132bb6973fbfbd611efdf9b15 in 1264ms, sequenceid=156, compaction requested=true 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:58,610 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:58,610 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:15:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:58,611 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:58,611 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:58,611 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:15:58,611 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:15:58,611 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,611 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,611 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/d19800e45ee34b02a450dda53f903932, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=47.3 K 2024-11-10T16:15:58,611 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=129.8 K 2024-11-10T16:15:58,612 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,612 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a] 2024-11-10T16:15:58,612 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef39c64a4be945a283ec10371a0ca555, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:58,612 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting d19800e45ee34b02a450dda53f903932, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:58,612 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb99afb6593342bcacedc7c4a373526a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255352382 2024-11-10T16:15:58,612 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting aee70aa163fc4ea4ac64779a7ddeda08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255352382 2024-11-10T16:15:58,613 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9036ab1cb6364cd3bac3f6653bf6697b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731255353988 2024-11-10T16:15:58,613 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 71bdbaabed454d13a2c3bcc1bae284ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731255353988 2024-11-10T16:15:58,613 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b991b23b91794427a7ccbd0427df863b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:15:58,613 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3b81c098aee4b93bd9105b6b9633a4a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:15:58,619 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:58,620 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:58,621 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/352cc7f4c4c44f62bb2e10a12856b60b is 50, key is test_row_0/B:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:58,622 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111052b1b941209049b187a12233265e1efb_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:58,625 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111052b1b941209049b187a12233265e1efb_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:58,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742366_1542 (size=12493) 2024-11-10T16:15:58,625 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111052b1b941209049b187a12233265e1efb_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:58,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742367_1543 (size=4469) 2024-11-10T16:15:58,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T16:15:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:58,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:15:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:58,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076f75ec5680a462facbfbf3c307f717c_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255357399/Put/seqid=0 2024-11-10T16:15:58,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742368_1544 (size=12304) 2024-11-10T16:15:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T16:15:59,030 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:59,030 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/352cc7f4c4c44f62bb2e10a12856b60b as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/352cc7f4c4c44f62bb2e10a12856b60b 2024-11-10T16:15:59,030 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0341e7499a544bd9943f06dcae51a8d2 is 175, key is test_row_0/A:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:59,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742369_1545 (size=31447) 2024-11-10T16:15:59,035 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 352cc7f4c4c44f62bb2e10a12856b60b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:59,035 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:59,035 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=12, startTime=1731255358610; duration=0sec 2024-11-10T16:15:59,035 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:15:59,035 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:15:59,035 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T16:15:59,037 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T16:15:59,038 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:15:59,038 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,038 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f13e43438cc414f98871ac8be8e8b9b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=47.3 K 2024-11-10T16:15:59,038 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f13e43438cc414f98871ac8be8e8b9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731255351213 2024-11-10T16:15:59,039 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b06537789a3409498cc069b45cfa1fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731255352382 2024-11-10T16:15:59,039 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting a823b8d0688742f39c770cef390ddfa1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731255353988 2024-11-10T16:15:59,039 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 64f5f2ca3f604d36bbe00445c8e93286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:15:59,040 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0341e7499a544bd9943f06dcae51a8d2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2 2024-11-10T16:15:59,045 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into 0341e7499a544bd9943f06dcae51a8d2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:59,045 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:59,045 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=12, startTime=1731255358610; duration=0sec 2024-11-10T16:15:59,045 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:59,045 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:15:59,048 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:15:59,048 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/b2d3ddde4b91471eb6706d4e2e7b4e25 is 50, key is test_row_0/C:col10/1731255356198/Put/seqid=0 2024-11-10T16:15:59,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742370_1546 (size=12493) 2024-11-10T16:15:59,055 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/b2d3ddde4b91471eb6706d4e2e7b4e25 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b2d3ddde4b91471eb6706d4e2e7b4e25 2024-11-10T16:15:59,059 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into b2d3ddde4b91471eb6706d4e2e7b4e25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:15:59,059 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:59,059 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=12, startTime=1731255358610; duration=0sec 2024-11-10T16:15:59,059 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:15:59,059 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:15:59,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,152 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111076f75ec5680a462facbfbf3c307f717c_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076f75ec5680a462facbfbf3c307f717c_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9593d0278e4a43c481c6af44b2cbe2f6, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:59,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9593d0278e4a43c481c6af44b2cbe2f6 is 175, key is test_row_0/A:col10/1731255357399/Put/seqid=0 2024-11-10T16:15:59,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742371_1547 (size=31105) 2024-11-10T16:15:59,174 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9593d0278e4a43c481c6af44b2cbe2f6 2024-11-10T16:15:59,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/98b0ead48fd0435fba9a7b781ae561d5 is 50, key is test_row_0/B:col10/1731255357399/Put/seqid=0 2024-11-10T16:15:59,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742372_1548 (size=12151) 2024-11-10T16:15:59,183 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/98b0ead48fd0435fba9a7b781ae561d5 2024-11-10T16:15:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/1e39cbdf316d49029ea1140268bd6348 is 50, key is test_row_0/C:col10/1731255357399/Put/seqid=0 2024-11-10T16:15:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742373_1549 (size=12151) 2024-11-10T16:15:59,195 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/1e39cbdf316d49029ea1140268bd6348 2024-11-10T16:15:59,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/9593d0278e4a43c481c6af44b2cbe2f6 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6 2024-11-10T16:15:59,202 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6, entries=150, sequenceid=168, filesize=30.4 K 2024-11-10T16:15:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/98b0ead48fd0435fba9a7b781ae561d5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5 2024-11-10T16:15:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T16:15:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/1e39cbdf316d49029ea1140268bd6348 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348 2024-11-10T16:15:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,211 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348, entries=150, sequenceid=168, filesize=11.9 K 2024-11-10T16:15:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,211 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 86e0154132bb6973fbfbd611efdf9b15 in 473ms, sequenceid=168, compaction requested=false 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-10T16:15:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T16:15:59,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-10T16:15:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0920 sec 2024-11-10T16:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T16:15:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,263 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T16:15:59,266 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,266 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T16:15:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T16:15:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:15:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T16:15:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T16:15:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T16:15:59,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-10T16:15:59,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 160 msec 2024-11-10T16:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,584 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:15:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:15:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,586 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:15:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,587 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:59,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T16:15:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:15:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:15:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:15:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:15:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043f90d939a634b4e81e648945302c67a_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255359603/Put/seqid=0 2024-11-10T16:15:59,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742374_1550 (size=22268) 2024-11-10T16:15:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,626 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,630 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111043f90d939a634b4e81e648945302c67a_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043f90d939a634b4e81e648945302c67a_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:15:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:15:59,631 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1cf5c90bea5b4ea9b674469d46cf6586, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:15:59,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1cf5c90bea5b4ea9b674469d46cf6586 is 175, key is test_row_0/A:col10/1731255359603/Put/seqid=0 2024-11-10T16:15:59,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742375_1551 (size=65673) 2024-11-10T16:15:59,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255359628, max timestamp=1731255359662Num uniques 859; 2024-11-10T16:15:59,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:59,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731255359628, max timestamp=1731255359662Num uniques 859; 2024-11-10T16:15:59,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T16:15:59,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255359628, max timestamp=1731255359662Num uniques 859; 2024-11-10T16:15:59,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:15:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:15:59,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255419713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:59,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255419717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:59,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:15:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:59,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255419823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:59,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:15:59,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255419833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:15:59,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:15:59,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:15:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:15:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:15:59,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:15:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:16:00,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255420033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,034 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,035 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1cf5c90bea5b4ea9b674469d46cf6586 2024-11-10T16:16:00,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/57892c1ae29744898b574cdc2daed1d8 is 50, key is test_row_0/B:col10/1731255359603/Put/seqid=0 2024-11-10T16:16:00,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:16:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255420043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742376_1552 (size=12151) 2024-11-10T16:16:00,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/57892c1ae29744898b574cdc2daed1d8 2024-11-10T16:16:00,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/aee4a3c2b7e3435dba9d2e49f381028d is 50, key is test_row_0/C:col10/1731255359603/Put/seqid=0 2024-11-10T16:16:00,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742377_1553 (size=12151) 2024-11-10T16:16:00,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:16:00,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255420343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:00,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255420353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/aee4a3c2b7e3435dba9d2e49f381028d 2024-11-10T16:16:00,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1cf5c90bea5b4ea9b674469d46cf6586 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586 2024-11-10T16:16:00,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586, entries=350, sequenceid=182, filesize=64.1 K 2024-11-10T16:16:00,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/57892c1ae29744898b574cdc2daed1d8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8 2024-11-10T16:16:00,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T16:16:00,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/aee4a3c2b7e3435dba9d2e49f381028d as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d 2024-11-10T16:16:00,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d, entries=150, sequenceid=182, filesize=11.9 K 2024-11-10T16:16:00,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 86e0154132bb6973fbfbd611efdf9b15 in 866ms, sequenceid=182, compaction requested=true 2024-11-10T16:16:00,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:16:00,478 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:16:00,478 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:00,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:00,479 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:00,479 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:16:00,479 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,479 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=125.2 K 2024-11-10T16:16:00,479 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,479 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586] 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0341e7499a544bd9943f06dcae51a8d2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9593d0278e4a43c481c6af44b2cbe2f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731255357351 2024-11-10T16:16:00,480 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,480 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/352cc7f4c4c44f62bb2e10a12856b60b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.9 K 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf5c90bea5b4ea9b674469d46cf6586, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359567 2024-11-10T16:16:00,480 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 352cc7f4c4c44f62bb2e10a12856b60b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:16:00,481 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b0ead48fd0435fba9a7b781ae561d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731255357351 2024-11-10T16:16:00,481 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 57892c1ae29744898b574cdc2daed1d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359603 2024-11-10T16:16:00,487 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:00,489 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:00,490 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/956de801b5fb4faabf9ae2201b4f7bc0 is 50, key is test_row_0/B:col10/1731255359603/Put/seqid=0 2024-11-10T16:16:00,491 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100b15a9d35f9446a6a966d5822e0b2ad5_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:00,494 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100b15a9d35f9446a6a966d5822e0b2ad5_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:00,494 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100b15a9d35f9446a6a966d5822e0b2ad5_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:00,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742378_1554 (size=12595) 2024-11-10T16:16:00,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742379_1555 (size=4469) 2024-11-10T16:16:00,502 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#464 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:00,503 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b69db342094e420e91260b4bba22769a is 175, key is test_row_0/A:col10/1731255359603/Put/seqid=0 2024-11-10T16:16:00,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-10T16:16:00,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:00,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:00,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742380_1556 (size=31549) 2024-11-10T16:16:00,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105e4d52d9ada84a1eb89c66b00883dab5_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255359703/Put/seqid=0 2024-11-10T16:16:00,514 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/b69db342094e420e91260b4bba22769a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a 2024-11-10T16:16:00,519 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into b69db342094e420e91260b4bba22769a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:00,519 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:00,519 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255360478; duration=0sec 2024-11-10T16:16:00,519 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:00,519 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:16:00,519 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:00,520 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:00,521 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:16:00,521 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:00,521 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b2d3ddde4b91471eb6706d4e2e7b4e25, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=35.9 K 2024-11-10T16:16:00,521 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d3ddde4b91471eb6706d4e2e7b4e25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731255356196 2024-11-10T16:16:00,521 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e39cbdf316d49029ea1140268bd6348, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731255357351 2024-11-10T16:16:00,521 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee4a3c2b7e3435dba9d2e49f381028d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359603 2024-11-10T16:16:00,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742381_1557 (size=12304) 2024-11-10T16:16:00,533 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:00,534 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/8a9c559e83714ba3a864b98cf357915e is 50, key is test_row_0/C:col10/1731255359603/Put/seqid=0 2024-11-10T16:16:00,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742382_1558 (size=12595) 2024-11-10T16:16:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:00,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255420596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255420597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255420599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255420713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255420713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255420713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:16:00,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255420853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255420863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,904 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/956de801b5fb4faabf9ae2201b4f7bc0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/956de801b5fb4faabf9ae2201b4f7bc0 2024-11-10T16:16:00,908 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 956de801b5fb4faabf9ae2201b4f7bc0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:00,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:00,909 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255360478; duration=0sec 2024-11-10T16:16:00,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:00,909 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:16:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255420923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255420924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255420924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:00,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:00,939 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105e4d52d9ada84a1eb89c66b00883dab5_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105e4d52d9ada84a1eb89c66b00883dab5_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:00,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/14cb5e74480f4de8906d29231021a4eb, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:00,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/14cb5e74480f4de8906d29231021a4eb is 175, key is test_row_0/A:col10/1731255359703/Put/seqid=0 2024-11-10T16:16:00,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742383_1559 (size=31105) 2024-11-10T16:16:00,944 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/14cb5e74480f4de8906d29231021a4eb 2024-11-10T16:16:00,948 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/8a9c559e83714ba3a864b98cf357915e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/8a9c559e83714ba3a864b98cf357915e 2024-11-10T16:16:00,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/0786176ad137479a9a5ee724bfde9571 is 50, key is test_row_0/B:col10/1731255359703/Put/seqid=0 2024-11-10T16:16:00,953 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into 8a9c559e83714ba3a864b98cf357915e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:00,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:00,953 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255360478; duration=0sec 2024-11-10T16:16:00,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:00,953 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:16:00,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742384_1560 (size=12151) 2024-11-10T16:16:00,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/0786176ad137479a9a5ee724bfde9571 2024-11-10T16:16:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/568cdfe6d9fd401ca655d42e46e41def is 50, key is test_row_0/C:col10/1731255359703/Put/seqid=0 2024-11-10T16:16:00,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742385_1561 (size=12151) 2024-11-10T16:16:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255421233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255421233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255421233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,365 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/568cdfe6d9fd401ca655d42e46e41def 2024-11-10T16:16:01,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/14cb5e74480f4de8906d29231021a4eb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb 2024-11-10T16:16:01,372 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb, entries=150, sequenceid=218, filesize=30.4 K 2024-11-10T16:16:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/0786176ad137479a9a5ee724bfde9571 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571 2024-11-10T16:16:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,376 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T16:16:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/568cdfe6d9fd401ca655d42e46e41def as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def 2024-11-10T16:16:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,380 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def, entries=150, sequenceid=218, filesize=11.9 K 2024-11-10T16:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,381 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for 86e0154132bb6973fbfbd611efdf9b15 in 877ms, sequenceid=218, compaction requested=false 2024-11-10T16:16:01,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-10T16:16:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-10T16:16:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T16:16:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,385 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7960 sec 2024-11-10T16:16:01,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,387 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.8020 sec 2024-11-10T16:16:01,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T16:16:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,734 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:01,734 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:16:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-10T16:16:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:16:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T16:16:01,738 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:16:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:16:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:01,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ed9b606f4d2b420f9d4b49228e3148a2_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742386_1562 (size=14794) 2024-11-10T16:16:01,784 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ed9b606f4d2b420f9d4b49228e3148a2_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed9b606f4d2b420f9d4b49228e3148a2_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:01,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/f27b576250d742eb9160d9f86e743ec3, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/f27b576250d742eb9160d9f86e743ec3 is 175, key is test_row_0/A:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255421787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255361767, max timestamp=1731255361791Num uniques 859; 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255361767, max timestamp=1731255361791Num uniques 859; 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731255361767, max timestamp=1731255361791Num uniques 859; 2024-11-10T16:16:01,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8e3de9160f9f:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T16:16:01,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742387_1563 (size=39745) 2024-11-10T16:16:01,812 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/f27b576250d742eb9160d9f86e743ec3 2024-11-10T16:16:01,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/781b1a4e1fb046f7a3f21e39a378c3db is 50, key is test_row_0/B:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742388_1564 (size=9757) 2024-11-10T16:16:01,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/781b1a4e1fb046f7a3f21e39a378c3db 2024-11-10T16:16:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T16:16:01,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7f4107967574470888650b2995342db8 is 50, key is test_row_0/C:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255421842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255421844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742389_1565 (size=9757) 2024-11-10T16:16:01,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7f4107967574470888650b2995342db8 2024-11-10T16:16:01,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/f27b576250d742eb9160d9f86e743ec3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3 2024-11-10T16:16:01,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3, entries=200, sequenceid=233, filesize=38.8 K 2024-11-10T16:16:01,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/781b1a4e1fb046f7a3f21e39a378c3db as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db 2024-11-10T16:16:01,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db, entries=100, sequenceid=233, filesize=9.5 K 2024-11-10T16:16:01,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7f4107967574470888650b2995342db8 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8 2024-11-10T16:16:01,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255421873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,879 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_0' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8, entries=100, sequenceid=233, filesize=9.5 K 2024-11-10T16:16:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 86e0154132bb6973fbfbd611efdf9b15 in 133ms, sequenceid=233, compaction requested=true 2024-11-10T16:16:01,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:01,881 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:01,882 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:01,882 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:16:01,883 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,883 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=100.0 K 2024-11-10T16:16:01,883 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,883 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3] 2024-11-10T16:16:01,883 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting b69db342094e420e91260b4bba22769a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359603 2024-11-10T16:16:01,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:16:01,884 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14cb5e74480f4de8906d29231021a4eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731255359697 2024-11-10T16:16:01,884 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting f27b576250d742eb9160d9f86e743ec3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255360595 2024-11-10T16:16:01,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:01,886 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:01,888 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:01,888 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:16:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:16:01,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:01,888 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,888 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/956de801b5fb4faabf9ae2201b4f7bc0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=33.7 K 2024-11-10T16:16:01,889 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 956de801b5fb4faabf9ae2201b4f7bc0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359603 2024-11-10T16:16:01,889 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0786176ad137479a9a5ee724bfde9571, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731255359697 2024-11-10T16:16:01,890 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 781b1a4e1fb046f7a3f21e39a378c3db, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255360595 2024-11-10T16:16:01,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-10T16:16:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:16:01,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:01,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:01,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T16:16:01,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:01,897 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,900 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:01,901 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/30c791d62f024022ab59d22b5730ffe5 is 50, key is test_row_0/B:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,916 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411109505f2aba3ae48b7a937a1e1db8068f5_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,918 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411109505f2aba3ae48b7a937a1e1db8068f5_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,918 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109505f2aba3ae48b7a937a1e1db8068f5_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103ac14be6096e42c9857f9f518e049642_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255361840/Put/seqid=0 2024-11-10T16:16:01,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742390_1566 (size=12697) 2024-11-10T16:16:01,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255421921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742391_1567 (size=4469) 2024-11-10T16:16:01,930 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#473 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:01,931 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ab6e9522721148798a8555de2bfc6d57 is 175, key is test_row_0/A:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255421924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,933 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/30c791d62f024022ab59d22b5730ffe5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/30c791d62f024022ab59d22b5730ffe5 2024-11-10T16:16:01,938 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 30c791d62f024022ab59d22b5730ffe5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:01,938 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:01,938 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255361886; duration=0sec 2024-11-10T16:16:01,938 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:01,938 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:16:01,938 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:01,939 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:01,940 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:16:01,940 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:01,940 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/8a9c559e83714ba3a864b98cf357915e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=33.7 K 2024-11-10T16:16:01,940 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9c559e83714ba3a864b98cf357915e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731255359603 2024-11-10T16:16:01,940 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 568cdfe6d9fd401ca655d42e46e41def, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731255359697 2024-11-10T16:16:01,941 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f4107967574470888650b2995342db8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255360595 2024-11-10T16:16:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742393_1569 (size=31758) 2024-11-10T16:16:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742392_1568 (size=12454) 2024-11-10T16:16:01,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:01,951 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103ac14be6096e42c9857f9f518e049642_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ac14be6096e42c9857f9f518e049642_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/5556bde2bb7d443194d73bbe3bd1f44a, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:01,952 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/5556bde2bb7d443194d73bbe3bd1f44a is 175, key is test_row_0/A:col10/1731255361840/Put/seqid=0 2024-11-10T16:16:01,954 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/ab6e9522721148798a8555de2bfc6d57 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57 2024-11-10T16:16:01,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742394_1570 (size=31255) 2024-11-10T16:16:01,956 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/5556bde2bb7d443194d73bbe3bd1f44a 2024-11-10T16:16:01,959 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into ab6e9522721148798a8555de2bfc6d57(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:01,959 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:01,959 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255361881; duration=0sec 2024-11-10T16:16:01,959 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:01,959 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:16:01,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255421953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255421953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,961 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/f74bc6223e67479ebbf5737de01a6521 is 50, key is test_row_0/C:col10/1731255361747/Put/seqid=0 2024-11-10T16:16:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:01,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5df5f7d77bb94f97bfa86d59e9289bb0 is 50, key is test_row_0/B:col10/1731255361840/Put/seqid=0 2024-11-10T16:16:01,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742395_1571 (size=12697) 2024-11-10T16:16:01,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742396_1572 (size=12301) 2024-11-10T16:16:01,974 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/f74bc6223e67479ebbf5737de01a6521 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f74bc6223e67479ebbf5737de01a6521 2024-11-10T16:16:01,976 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5df5f7d77bb94f97bfa86d59e9289bb0 2024-11-10T16:16:01,982 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into f74bc6223e67479ebbf5737de01a6521(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:01,982 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:01,982 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255361890; duration=0sec 2024-11-10T16:16:01,983 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:01,983 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:16:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7cad7990aee34100bc693ab74e29aed2 is 50, key is test_row_0/C:col10/1731255361840/Put/seqid=0 2024-11-10T16:16:01,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742397_1573 (size=12301) 2024-11-10T16:16:02,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255422033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255422043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T16:16:02,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255422163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255422163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255422243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255422253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T16:16:02,391 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7cad7990aee34100bc693ab74e29aed2 2024-11-10T16:16:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/5556bde2bb7d443194d73bbe3bd1f44a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a 2024-11-10T16:16:02,398 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a, entries=150, sequenceid=269, filesize=30.5 K 2024-11-10T16:16:02,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5df5f7d77bb94f97bfa86d59e9289bb0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0 2024-11-10T16:16:02,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0, entries=150, sequenceid=269, filesize=12.0 K 2024-11-10T16:16:02,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/7cad7990aee34100bc693ab74e29aed2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2 2024-11-10T16:16:02,405 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2, entries=150, sequenceid=269, filesize=12.0 K 2024-11-10T16:16:02,406 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 86e0154132bb6973fbfbd611efdf9b15 in 515ms, sequenceid=269, compaction requested=false 2024-11-10T16:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:02,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-10T16:16:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-10T16:16:02,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T16:16:02,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 670 msec 2024-11-10T16:16:02,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 676 msec 2024-11-10T16:16:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:02,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:02,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a965a859bf04fc4bd05e5bfaa268788_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:02,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742398_1574 (size=17534) 2024-11-10T16:16:02,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255422520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255422521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255422553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255422563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255422633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255422633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,637 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255422843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255422843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:02,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-10T16:16:02,873 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:02,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:16:02,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-10T16:16:02,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:02,875 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:16:02,876 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:16:02,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:16:02,894 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:02,898 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411108a965a859bf04fc4bd05e5bfaa268788_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a965a859bf04fc4bd05e5bfaa268788_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:02,898 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/746716643b834c8c9f85abcc9feb6af5, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:02,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/746716643b834c8c9f85abcc9feb6af5 is 175, key is test_row_0/A:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742399_1575 (size=48639) 2024-11-10T16:16:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:03,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255423063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255423073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,076 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255423153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255423153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:03,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/746716643b834c8c9f85abcc9feb6af5 2024-11-10T16:16:03,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/af4a477d282949e8b2611e709caf6668 is 50, key is test_row_0/B:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:03,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742400_1576 (size=12301) 2024-11-10T16:16:03,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/af4a477d282949e8b2611e709caf6668 2024-11-10T16:16:03,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/3fc9ebaac2964398af231d596a494171 is 50, key is test_row_0/C:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:03,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742401_1577 (size=12301) 2024-11-10T16:16:03,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:03,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:03,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255423663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255423664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:03,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/3fc9ebaac2964398af231d596a494171 2024-11-10T16:16:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/746716643b834c8c9f85abcc9feb6af5 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5 2024-11-10T16:16:03,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5, entries=250, sequenceid=285, filesize=47.5 K 2024-11-10T16:16:03,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/af4a477d282949e8b2611e709caf6668 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668 2024-11-10T16:16:03,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T16:16:03,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/3fc9ebaac2964398af231d596a494171 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171 2024-11-10T16:16:03,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171, entries=150, sequenceid=285, filesize=12.0 K 2024-11-10T16:16:03,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 86e0154132bb6973fbfbd611efdf9b15 in 1261ms, sequenceid=285, compaction requested=true 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:03,737 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:16:03,737 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:03,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:03,738 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:03,738 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111652 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:03,738 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:16:03,738 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:16:03,738 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,738 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,738 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/30c791d62f024022ab59d22b5730ffe5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=36.4 K 2024-11-10T16:16:03,738 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=109.0 K 2024-11-10T16:16:03,738 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,738 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5] 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c791d62f024022ab59d22b5730ffe5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255359716 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6e9522721148798a8555de2bfc6d57, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255359716 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 5df5f7d77bb94f97bfa86d59e9289bb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731255361840 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5556bde2bb7d443194d73bbe3bd1f44a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731255361840 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting af4a477d282949e8b2611e709caf6668, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:03,739 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746716643b834c8c9f85abcc9feb6af5, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:03,745 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:03,745 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/14884d96c5fc4f74895a41b222f967ee is 50, key is test_row_0/B:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:03,746 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:03,748 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110d8e4c975e03a49bf9b535ba8702e0d05_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742402_1578 (size=12949) 2024-11-10T16:16:03,750 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110d8e4c975e03a49bf9b535ba8702e0d05_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:03,751 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d8e4c975e03a49bf9b535ba8702e0d05_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:03,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742403_1579 (size=4469) 2024-11-10T16:16:03,755 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/14884d96c5fc4f74895a41b222f967ee as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/14884d96c5fc4f74895a41b222f967ee 2024-11-10T16:16:03,756 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#483 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:03,756 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/d4bc10de36ae42058c33df9a510f7a93 is 175, key is test_row_0/A:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:03,759 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 14884d96c5fc4f74895a41b222f967ee(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:03,759 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:03,759 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255363737; duration=0sec 2024-11-10T16:16:03,759 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:03,760 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:16:03,760 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:03,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742404_1580 (size=31903) 2024-11-10T16:16:03,761 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:03,761 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:16:03,761 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,761 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f74bc6223e67479ebbf5737de01a6521, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=36.4 K 2024-11-10T16:16:03,761 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f74bc6223e67479ebbf5737de01a6521, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731255359716 2024-11-10T16:16:03,761 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cad7990aee34100bc693ab74e29aed2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731255361840 2024-11-10T16:16:03,762 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fc9ebaac2964398af231d596a494171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:03,764 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/d4bc10de36ae42058c33df9a510f7a93 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93 2024-11-10T16:16:03,769 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:03,769 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/0f0a0336926b473cb035cf5ea4de0fa3 is 50, key is test_row_0/C:col10/1731255361915/Put/seqid=0 2024-11-10T16:16:03,771 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into d4bc10de36ae42058c33df9a510f7a93(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:03,771 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:03,771 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255363737; duration=0sec 2024-11-10T16:16:03,771 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:03,771 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:16:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742405_1581 (size=12949) 2024-11-10T16:16:03,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-10T16:16:03,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:03,795 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:03,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101dcb3b8bce2344789cbc29a0ddfeb7e7_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255362501/Put/seqid=0 2024-11-10T16:16:03,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742406_1582 (size=12454) 2024-11-10T16:16:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:03,807 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101dcb3b8bce2344789cbc29a0ddfeb7e7_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101dcb3b8bce2344789cbc29a0ddfeb7e7_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/979f101b9e94428696f49c0dd0119646, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:03,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/979f101b9e94428696f49c0dd0119646 is 175, key is test_row_0/A:col10/1731255362501/Put/seqid=0 2024-11-10T16:16:03,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742407_1583 (size=31255) 2024-11-10T16:16:03,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:03,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:03,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255423939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,943 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:04,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255424053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255424073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,076 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255424083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,087 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,177 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/0f0a0336926b473cb035cf5ea4de0fa3 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/0f0a0336926b473cb035cf5ea4de0fa3 2024-11-10T16:16:04,181 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into 0f0a0336926b473cb035cf5ea4de0fa3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:04,181 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:04,181 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255363737; duration=0sec 2024-11-10T16:16:04,181 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:04,181 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:16:04,212 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/979f101b9e94428696f49c0dd0119646 2024-11-10T16:16:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/17633c98a62242c6a56bbdf99b5252e2 is 50, key is test_row_0/B:col10/1731255362501/Put/seqid=0 2024-11-10T16:16:04,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742408_1584 (size=12301) 2024-11-10T16:16:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255424263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255424573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,575 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/17633c98a62242c6a56bbdf99b5252e2 2024-11-10T16:16:04,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/f2f82571b9204984a1a36d939bcbf4a9 is 50, key is test_row_0/C:col10/1731255362501/Put/seqid=0 2024-11-10T16:16:04,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742409_1585 (size=12301) 2024-11-10T16:16:04,638 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/f2f82571b9204984a1a36d939bcbf4a9 2024-11-10T16:16:04,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/979f101b9e94428696f49c0dd0119646 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646 2024-11-10T16:16:04,646 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646, entries=150, sequenceid=306, filesize=30.5 K 2024-11-10T16:16:04,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/17633c98a62242c6a56bbdf99b5252e2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2 2024-11-10T16:16:04,650 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T16:16:04,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/f2f82571b9204984a1a36d939bcbf4a9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9 2024-11-10T16:16:04,653 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9, entries=150, sequenceid=306, filesize=12.0 K 2024-11-10T16:16:04,654 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 86e0154132bb6973fbfbd611efdf9b15 in 860ms, sequenceid=306, compaction requested=false 2024-11-10T16:16:04,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:04,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:04,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-10T16:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-10T16:16:04,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-10T16:16:04,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-11-10T16:16:04,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.7840 sec 2024-11-10T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:04,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:04,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:04,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd32b7f706aa49e185269919eab2b5d8_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:04,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742410_1586 (size=12454) 2024-11-10T16:16:04,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255424725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255424727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255424833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:04,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:04,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255424843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:04,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T16:16:05,023 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:05,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T16:16:05,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-10T16:16:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:05,025 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T16:16:05,026 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T16:16:05,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T16:16:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255425043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255425053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255425083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,108 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:05,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110cd32b7f706aa49e185269919eab2b5d8_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd32b7f706aa49e185269919eab2b5d8_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:05,112 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:05,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 is 175, key is test_row_0/A:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:05,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742411_1587 (size=31255) 2024-11-10T16:16:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:05,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:05,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255425353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255425363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 2024-11-10T16:16:05,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/afff0f535a164422adc1a8083becca82 is 50, key is test_row_0/B:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:05,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742412_1588 (size=12301) 2024-11-10T16:16:05,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,647 INFO [Thread-2259 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2259. 2024-11-10T16:16:05,647 INFO [Thread-2265 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2265. 2024-11-10T16:16:05,647 DEBUG [Thread-2259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:05,647 DEBUG [Thread-2265 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:05,647 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,647 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,647 DEBUG [Thread-2259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,647 DEBUG [Thread-2265 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:05,647 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:05,648 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-10T16:16:05,648 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:05,648 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,648 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:05,651 INFO [Thread-2261 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2261. 2024-11-10T16:16:05,651 DEBUG [Thread-2261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:05,651 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,651 DEBUG [Thread-2261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:05,651 INFO [Thread-2263 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2263. 2024-11-10T16:16:05,651 DEBUG [Thread-2263 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:05,651 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,652 DEBUG [Thread-2263 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:05,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:05,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58020 deadline: 1731255425863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57988 deadline: 1731255425873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:05,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:05,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/afff0f535a164422adc1a8083becca82 2024-11-10T16:16:05,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/b5516e09beb04c5e95908c154b316ebe is 50, key is test_row_0/C:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:05,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742413_1589 (size=12301) 2024-11-10T16:16:05,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:05,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:05,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:05,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57980 deadline: 1731255426083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:06,084 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:06,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58008 deadline: 1731255426093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:06,093 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_2' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:06,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T16:16:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58018 deadline: 1731255426103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:06,103 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 8e3de9160f9f,44105,1731255200149 for 'test_row_1' in 86e0154132bb6973fbfbd611efdf9b15 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T16:16:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 , the old value is region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T16:16:06,104 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., hostname=8e3de9160f9f,44105,1731255200149, seqNum=5 because the exception is null or not the one we care about 2024-11-10T16:16:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:06,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T16:16:06,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/b5516e09beb04c5e95908c154b316ebe 2024-11-10T16:16:06,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 2024-11-10T16:16:06,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9, entries=150, sequenceid=321, filesize=30.5 K 2024-11-10T16:16:06,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/afff0f535a164422adc1a8083becca82 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82 2024-11-10T16:16:06,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T16:16:06,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/b5516e09beb04c5e95908c154b316ebe as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe 2024-11-10T16:16:06,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe, entries=150, sequenceid=321, filesize=12.0 K 2024-11-10T16:16:06,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 86e0154132bb6973fbfbd611efdf9b15 in 1671ms, sequenceid=321, compaction requested=true 2024-11-10T16:16:06,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 86e0154132bb6973fbfbd611efdf9b15:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:06,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/A is initiating minor compaction (all files) 2024-11-10T16:16:06,348 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/B is initiating minor compaction (all files) 2024-11-10T16:16:06,348 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/A in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,348 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/B in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,349 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=92.2 K 2024-11-10T16:16:06,349 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/14884d96c5fc4f74895a41b222f967ee, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=36.7 K 2024-11-10T16:16:06,349 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. files: [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9] 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 14884d96c5fc4f74895a41b222f967ee, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4bc10de36ae42058c33df9a510f7a93, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 17633c98a62242c6a56bbdf99b5252e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255362501 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 979f101b9e94428696f49c0dd0119646, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255362501 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting afff0f535a164422adc1a8083becca82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255363931 2024-11-10T16:16:06,349 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a7e9bd2145c42ce8ce3c756c3f0c0d9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255363931 2024-11-10T16:16:06,354 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:06,355 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#B#compaction#491 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:06,355 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110a99df2a84cbe4d178ee6ac38db5ee0aa_86e0154132bb6973fbfbd611efdf9b15 store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:06,356 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/6bd4dea8eb7f46fb9aac304ee97b5d35 is 50, key is test_row_0/B:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:06,359 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110a99df2a84cbe4d178ee6ac38db5ee0aa_86e0154132bb6973fbfbd611efdf9b15, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:06,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742414_1590 (size=13051) 2024-11-10T16:16:06,359 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a99df2a84cbe4d178ee6ac38db5ee0aa_86e0154132bb6973fbfbd611efdf9b15 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:06,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742415_1591 (size=4469) 2024-11-10T16:16:06,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44105 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,400 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:06,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:06,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111018e2e77ca931478495e53eef6719538e_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255364726/Put/seqid=0 2024-11-10T16:16:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742416_1592 (size=12454) 2024-11-10T16:16:06,762 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#A#compaction#492 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:06,763 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/6bd4dea8eb7f46fb9aac304ee97b5d35 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/6bd4dea8eb7f46fb9aac304ee97b5d35 2024-11-10T16:16:06,763 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c3f6e427d2ac417d8d26d94ebec2110a is 175, key is test_row_0/A:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:06,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742417_1593 (size=32005) 2024-11-10T16:16:06,767 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/B of 86e0154132bb6973fbfbd611efdf9b15 into 6bd4dea8eb7f46fb9aac304ee97b5d35(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:06,767 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:06,767 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/B, priority=13, startTime=1731255366348; duration=0sec 2024-11-10T16:16:06,767 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T16:16:06,767 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:B 2024-11-10T16:16:06,767 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T16:16:06,768 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T16:16:06,768 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1541): 86e0154132bb6973fbfbd611efdf9b15/C is initiating minor compaction (all files) 2024-11-10T16:16:06,768 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 86e0154132bb6973fbfbd611efdf9b15/C in TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:06,768 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/0f0a0336926b473cb035cf5ea4de0fa3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe] into tmpdir=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp, totalSize=36.7 K 2024-11-10T16:16:06,768 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f0a0336926b473cb035cf5ea4de0fa3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731255361915 2024-11-10T16:16:06,768 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting f2f82571b9204984a1a36d939bcbf4a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731255362501 2024-11-10T16:16:06,768 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] compactions.Compactor(225): Compacting b5516e09beb04c5e95908c154b316ebe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731255363931 2024-11-10T16:16:06,775 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 86e0154132bb6973fbfbd611efdf9b15#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T16:16:06,775 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/5ac78c0159c04654aaf3ffa4da99b60a is 50, key is test_row_0/C:col10/1731255363931/Put/seqid=0 2024-11-10T16:16:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742418_1594 (size=13051) 2024-11-10T16:16:06,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:06,811 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111018e2e77ca931478495e53eef6719538e_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018e2e77ca931478495e53eef6719538e_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1594778186c24f44b26f16c0919e500f, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1594778186c24f44b26f16c0919e500f is 175, key is test_row_0/A:col10/1731255364726/Put/seqid=0 2024-11-10T16:16:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742419_1595 (size=31255) 2024-11-10T16:16:06,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44105 {}] regionserver.HRegion(8855): Flush requested on 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:06,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. as already flushing 2024-11-10T16:16:06,875 INFO [Thread-2248 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2248. 2024-11-10T16:16:06,875 DEBUG [Thread-2248 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:06,875 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:06,875 DEBUG [Thread-2248 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:06,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:06,884 INFO [Thread-2252 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2252. 2024-11-10T16:16:06,884 DEBUG [Thread-2252 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:06,884 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:06,885 DEBUG [Thread-2252 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:06,885 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:07,170 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/c3f6e427d2ac417d8d26d94ebec2110a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c3f6e427d2ac417d8d26d94ebec2110a 2024-11-10T16:16:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:07,173 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/A of 86e0154132bb6973fbfbd611efdf9b15 into c3f6e427d2ac417d8d26d94ebec2110a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:07,174 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:07,174 INFO [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/A, priority=13, startTime=1731255366348; duration=0sec 2024-11-10T16:16:07,174 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:07,174 DEBUG [RS:0;8e3de9160f9f:44105-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:A 2024-11-10T16:16:07,181 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/5ac78c0159c04654aaf3ffa4da99b60a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/5ac78c0159c04654aaf3ffa4da99b60a 2024-11-10T16:16:07,184 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 86e0154132bb6973fbfbd611efdf9b15/C of 86e0154132bb6973fbfbd611efdf9b15 into 5ac78c0159c04654aaf3ffa4da99b60a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T16:16:07,184 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:07,184 INFO [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., storeName=86e0154132bb6973fbfbd611efdf9b15/C, priority=13, startTime=1731255366348; duration=0sec 2024-11-10T16:16:07,185 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T16:16:07,185 DEBUG [RS:0;8e3de9160f9f:44105-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 86e0154132bb6973fbfbd611efdf9b15:C 2024-11-10T16:16:07,215 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=339, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1594778186c24f44b26f16c0919e500f 2024-11-10T16:16:07,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/c7cf5d7f8c4740ff8be7ad214e8c77da is 50, key is test_row_0/B:col10/1731255364726/Put/seqid=0 2024-11-10T16:16:07,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742420_1596 (size=12301) 2024-11-10T16:16:07,624 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/c7cf5d7f8c4740ff8be7ad214e8c77da 2024-11-10T16:16:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/2f48022f03b8475e87630d56eb58d6f2 is 50, key is test_row_0/C:col10/1731255364726/Put/seqid=0 2024-11-10T16:16:07,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742421_1597 (size=12301) 2024-11-10T16:16:08,033 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/2f48022f03b8475e87630d56eb58d6f2 2024-11-10T16:16:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/1594778186c24f44b26f16c0919e500f as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1594778186c24f44b26f16c0919e500f 2024-11-10T16:16:08,039 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1594778186c24f44b26f16c0919e500f, entries=150, sequenceid=339, filesize=30.5 K 2024-11-10T16:16:08,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/c7cf5d7f8c4740ff8be7ad214e8c77da as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/c7cf5d7f8c4740ff8be7ad214e8c77da 2024-11-10T16:16:08,042 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/c7cf5d7f8c4740ff8be7ad214e8c77da, entries=150, sequenceid=339, filesize=12.0 K 2024-11-10T16:16:08,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/2f48022f03b8475e87630d56eb58d6f2 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/2f48022f03b8475e87630d56eb58d6f2 2024-11-10T16:16:08,045 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/2f48022f03b8475e87630d56eb58d6f2, entries=150, sequenceid=339, filesize=12.0 K 2024-11-10T16:16:08,046 INFO [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=13.42 KB/13740 for 86e0154132bb6973fbfbd611efdf9b15 in 1646ms, sequenceid=339, compaction requested=false 2024-11-10T16:16:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 86e0154132bb6973fbfbd611efdf9b15: 2024-11-10T16:16:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:08,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8e3de9160f9f:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-10T16:16:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-10T16:16:08,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-10T16:16:08,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0210 sec 2024-11-10T16:16:08,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.0250 sec 2024-11-10T16:16:08,114 INFO [Thread-2250 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2250. 2024-11-10T16:16:08,114 DEBUG [Thread-2250 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:08,114 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:08,115 DEBUG [Thread-2250 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:08,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:09,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-10T16:16:09,193 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:10,104 INFO [Thread-2254 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2254. 2024-11-10T16:16:10,104 DEBUG [Thread-2254 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:10,104 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,104 DEBUG [Thread-2254 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:10,125 INFO [Thread-2256 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2256. 2024-11-10T16:16:10,125 DEBUG [Thread-2256 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T16:16:10,125 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,125 DEBUG [Thread-2256 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2847 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2875 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2958 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2868 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2871 2024-11-10T16:16:10,125 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T16:16:10,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:16:10,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:16:10,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:10,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:10,126 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T16:16:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:10,128 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255370128"}]},"ts":"1731255370128"} 2024-11-10T16:16:10,129 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T16:16:10,130 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T16:16:10,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T16:16:10,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, UNASSIGN}] 2024-11-10T16:16:10,132 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, UNASSIGN 2024-11-10T16:16:10,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=CLOSING, regionLocation=8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:10,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, UNASSIGN because future has completed 2024-11-10T16:16:10,133 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T16:16:10,133 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149}] 2024-11-10T16:16:10,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:10,285 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:10,285 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T16:16:10,285 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 86e0154132bb6973fbfbd611efdf9b15, disabling compactions & flushes 2024-11-10T16:16:10,286 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. after waiting 0 ms 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:10,286 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 86e0154132bb6973fbfbd611efdf9b15 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=A 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=B 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 86e0154132bb6973fbfbd611efdf9b15, store=C 2024-11-10T16:16:10,286 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T16:16:10,290 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd96f1072f434f58a8921d89be74d660_86e0154132bb6973fbfbd611efdf9b15 is 50, key is test_row_0/A:col10/1731255366874/Put/seqid=0 2024-11-10T16:16:10,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742422_1598 (size=12454) 2024-11-10T16:16:10,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:10,694 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T16:16:10,697 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dd96f1072f434f58a8921d89be74d660_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd96f1072f434f58a8921d89be74d660_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:10,698 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/8b9e9e1a29b249f6af31724fd873f5c4, store: [table=TestAcidGuarantees family=A region=86e0154132bb6973fbfbd611efdf9b15] 2024-11-10T16:16:10,698 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/8b9e9e1a29b249f6af31724fd873f5c4 is 175, key is test_row_0/A:col10/1731255366874/Put/seqid=0 2024-11-10T16:16:10,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742423_1599 (size=31255) 2024-11-10T16:16:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:11,102 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/8b9e9e1a29b249f6af31724fd873f5c4 2024-11-10T16:16:11,108 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5e662befb7784b0fbee7b9c09bfff86e is 50, key is test_row_0/B:col10/1731255366874/Put/seqid=0 2024-11-10T16:16:11,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742424_1600 (size=12301) 2024-11-10T16:16:11,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:11,511 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5e662befb7784b0fbee7b9c09bfff86e 2024-11-10T16:16:11,517 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/c344750c900f47cea12b96623027a6dd is 50, key is test_row_0/C:col10/1731255366874/Put/seqid=0 2024-11-10T16:16:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742425_1601 (size=12301) 2024-11-10T16:16:11,920 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/c344750c900f47cea12b96623027a6dd 2024-11-10T16:16:11,924 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/A/8b9e9e1a29b249f6af31724fd873f5c4 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/8b9e9e1a29b249f6af31724fd873f5c4 2024-11-10T16:16:11,926 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/8b9e9e1a29b249f6af31724fd873f5c4, entries=150, sequenceid=350, filesize=30.5 K 2024-11-10T16:16:11,927 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/B/5e662befb7784b0fbee7b9c09bfff86e as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5e662befb7784b0fbee7b9c09bfff86e 2024-11-10T16:16:11,929 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5e662befb7784b0fbee7b9c09bfff86e, entries=150, sequenceid=350, filesize=12.0 K 2024-11-10T16:16:11,930 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/.tmp/C/c344750c900f47cea12b96623027a6dd as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/c344750c900f47cea12b96623027a6dd 2024-11-10T16:16:11,932 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/c344750c900f47cea12b96623027a6dd, entries=150, sequenceid=350, filesize=12.0 K 2024-11-10T16:16:11,933 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 86e0154132bb6973fbfbd611efdf9b15 in 1647ms, sequenceid=350, compaction requested=true 2024-11-10T16:16:11,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9] to archive 2024-11-10T16:16:11,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:16:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b8910e3d618047e1b2c390146a862ed3 2024-11-10T16:16:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a39f7c79d5264c2084ab1954d0edcd08 2024-11-10T16:16:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1e178ece65fb4f90afaefd30c818c519 2024-11-10T16:16:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b7bfa2142c1b4d8a927565a4c6e95c69 2024-11-10T16:16:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ad9b8dd29a2c4c96a26296812bfa3dce 2024-11-10T16:16:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c9019302d1b64a5aaeda5e414f83cace 2024-11-10T16:16:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ef39c64a4be945a283ec10371a0ca555 2024-11-10T16:16:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/fb99afb6593342bcacedc7c4a373526a 2024-11-10T16:16:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9036ab1cb6364cd3bac3f6653bf6697b 2024-11-10T16:16:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/a3b81c098aee4b93bd9105b6b9633a4a 2024-11-10T16:16:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0341e7499a544bd9943f06dcae51a8d2 2024-11-10T16:16:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/9593d0278e4a43c481c6af44b2cbe2f6 2024-11-10T16:16:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1cf5c90bea5b4ea9b674469d46cf6586 2024-11-10T16:16:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/b69db342094e420e91260b4bba22769a 2024-11-10T16:16:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/14cb5e74480f4de8906d29231021a4eb 2024-11-10T16:16:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/f27b576250d742eb9160d9f86e743ec3 2024-11-10T16:16:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/ab6e9522721148798a8555de2bfc6d57 2024-11-10T16:16:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/5556bde2bb7d443194d73bbe3bd1f44a 2024-11-10T16:16:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/746716643b834c8c9f85abcc9feb6af5 2024-11-10T16:16:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/d4bc10de36ae42058c33df9a510f7a93 2024-11-10T16:16:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/979f101b9e94428696f49c0dd0119646 2024-11-10T16:16:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/0a7e9bd2145c42ce8ce3c756c3f0c0d9 2024-11-10T16:16:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/53eccdb467534eb0bc207deb41528b13, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/d19800e45ee34b02a450dda53f903932, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/352cc7f4c4c44f62bb2e10a12856b60b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/956de801b5fb4faabf9ae2201b4f7bc0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/30c791d62f024022ab59d22b5730ffe5, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/14884d96c5fc4f74895a41b222f967ee, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82] to archive 2024-11-10T16:16:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:16:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/fe49ec9f71e6490a8bca08a5894cc7fb 2024-11-10T16:16:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/9b8d66799e1b49109eff1bff6aecc96a 2024-11-10T16:16:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/53eccdb467534eb0bc207deb41528b13 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/53eccdb467534eb0bc207deb41528b13 2024-11-10T16:16:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/eda0db07e7974d41912c4b623a01b401 2024-11-10T16:16:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5bd35bfe7f754fc69f64599dcd0cf909 2024-11-10T16:16:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/d19800e45ee34b02a450dda53f903932 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/d19800e45ee34b02a450dda53f903932 2024-11-10T16:16:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/cd1b96b02d37462ca706c75ece96bc0d 2024-11-10T16:16:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/aee70aa163fc4ea4ac64779a7ddeda08 2024-11-10T16:16:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/71bdbaabed454d13a2c3bcc1bae284ea 2024-11-10T16:16:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/352cc7f4c4c44f62bb2e10a12856b60b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/352cc7f4c4c44f62bb2e10a12856b60b 2024-11-10T16:16:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/b991b23b91794427a7ccbd0427df863b 2024-11-10T16:16:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/98b0ead48fd0435fba9a7b781ae561d5 2024-11-10T16:16:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/956de801b5fb4faabf9ae2201b4f7bc0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/956de801b5fb4faabf9ae2201b4f7bc0 2024-11-10T16:16:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/57892c1ae29744898b574cdc2daed1d8 2024-11-10T16:16:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/0786176ad137479a9a5ee724bfde9571 2024-11-10T16:16:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/30c791d62f024022ab59d22b5730ffe5 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/30c791d62f024022ab59d22b5730ffe5 2024-11-10T16:16:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/781b1a4e1fb046f7a3f21e39a378c3db 2024-11-10T16:16:11,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5df5f7d77bb94f97bfa86d59e9289bb0 2024-11-10T16:16:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/14884d96c5fc4f74895a41b222f967ee to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/14884d96c5fc4f74895a41b222f967ee 2024-11-10T16:16:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/af4a477d282949e8b2611e709caf6668 2024-11-10T16:16:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/17633c98a62242c6a56bbdf99b5252e2 2024-11-10T16:16:11,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/afff0f535a164422adc1a8083becca82 2024-11-10T16:16:11,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/62a0667f7d46432ebdf2129b68343633, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f13e43438cc414f98871ac8be8e8b9b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b2d3ddde4b91471eb6706d4e2e7b4e25, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/8a9c559e83714ba3a864b98cf357915e, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f74bc6223e67479ebbf5737de01a6521, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/0f0a0336926b473cb035cf5ea4de0fa3, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe] to archive 2024-11-10T16:16:11,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T16:16:11,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/05ef826758bc44848a9d13a90a1c8390 2024-11-10T16:16:11,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4f7d681a24dd4045be35a2cad2908cae 2024-11-10T16:16:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/62a0667f7d46432ebdf2129b68343633 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/62a0667f7d46432ebdf2129b68343633 2024-11-10T16:16:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a25fcbaf6f0045ce8cf06cb5942375e8 2024-11-10T16:16:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/32757b09c0544820b63d69b638c721b1 2024-11-10T16:16:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f13e43438cc414f98871ac8be8e8b9b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f13e43438cc414f98871ac8be8e8b9b 2024-11-10T16:16:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/92e79696f8fa45df8dc1ce7c60ddcb6b 2024-11-10T16:16:11,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/4b06537789a3409498cc069b45cfa1fd 2024-11-10T16:16:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/a823b8d0688742f39c770cef390ddfa1 2024-11-10T16:16:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b2d3ddde4b91471eb6706d4e2e7b4e25 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b2d3ddde4b91471eb6706d4e2e7b4e25 2024-11-10T16:16:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/64f5f2ca3f604d36bbe00445c8e93286 2024-11-10T16:16:11,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/1e39cbdf316d49029ea1140268bd6348 2024-11-10T16:16:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/8a9c559e83714ba3a864b98cf357915e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/8a9c559e83714ba3a864b98cf357915e 2024-11-10T16:16:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/aee4a3c2b7e3435dba9d2e49f381028d 2024-11-10T16:16:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/568cdfe6d9fd401ca655d42e46e41def 2024-11-10T16:16:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f74bc6223e67479ebbf5737de01a6521 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f74bc6223e67479ebbf5737de01a6521 2024-11-10T16:16:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7f4107967574470888650b2995342db8 2024-11-10T16:16:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/7cad7990aee34100bc693ab74e29aed2 2024-11-10T16:16:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/0f0a0336926b473cb035cf5ea4de0fa3 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/0f0a0336926b473cb035cf5ea4de0fa3 2024-11-10T16:16:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/3fc9ebaac2964398af231d596a494171 2024-11-10T16:16:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/f2f82571b9204984a1a36d939bcbf4a9 2024-11-10T16:16:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/b5516e09beb04c5e95908c154b316ebe 2024-11-10T16:16:11,994 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits/353.seqid, newMaxSeqId=353, maxSeqId=4 2024-11-10T16:16:11,994 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. 2024-11-10T16:16:11,994 DEBUG [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 86e0154132bb6973fbfbd611efdf9b15: Waiting for close lock at 1731255370285Running coprocessor pre-close hooks at 1731255370285Disabling compacts and flushes for region at 1731255370285Disabling writes for close at 1731255370286 (+1 ms)Obtaining lock to block concurrent updates at 1731255370286Preparing flush snapshotting stores in 86e0154132bb6973fbfbd611efdf9b15 at 1731255370286Finished memstore snapshotting TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731255370286Flushing stores of TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15. at 1731255370286Flushing 86e0154132bb6973fbfbd611efdf9b15/A: creating writer at 1731255370286Flushing mob file 86e0154132bb6973fbfbd611efdf9b15/A: appending metadata at 1731255370290 (+4 ms)Flushing mob file 86e0154132bb6973fbfbd611efdf9b15/A: closing flushed file at 1731255370290Flushing 86e0154132bb6973fbfbd611efdf9b15/A: appending metadata at 1731255370698 (+408 ms)Flushing 86e0154132bb6973fbfbd611efdf9b15/A: closing flushed file at 1731255370698Flushing 86e0154132bb6973fbfbd611efdf9b15/B: creating writer at 1731255371105 (+407 ms)Flushing 86e0154132bb6973fbfbd611efdf9b15/B: appending metadata at 1731255371107 (+2 ms)Flushing 86e0154132bb6973fbfbd611efdf9b15/B: closing flushed file at 1731255371107Flushing 86e0154132bb6973fbfbd611efdf9b15/C: creating writer at 1731255371514 (+407 ms)Flushing 86e0154132bb6973fbfbd611efdf9b15/C: appending metadata at 1731255371516 (+2 ms)Flushing 86e0154132bb6973fbfbd611efdf9b15/C: closing flushed file at 1731255371517 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b5867b9: reopening flushed file at 1731255371923 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72282d82: reopening flushed file at 1731255371926 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50fb862f: reopening flushed file at 1731255371929 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 86e0154132bb6973fbfbd611efdf9b15 in 1647ms, sequenceid=350, compaction requested=true at 1731255371933 (+4 ms)Writing region close event to WAL at 1731255371991 (+58 ms)Running coprocessor post-close hooks at 1731255371994 (+3 ms)Closed at 1731255371994 2024-11-10T16:16:11,996 INFO [RS_CLOSE_REGION-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:11,996 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=86e0154132bb6973fbfbd611efdf9b15, regionState=CLOSED 2024-11-10T16:16:11,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 because future has completed 2024-11-10T16:16:11,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-10T16:16:12,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 86e0154132bb6973fbfbd611efdf9b15, server=8e3de9160f9f,44105,1731255200149 in 1.8650 sec 2024-11-10T16:16:12,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-10T16:16:12,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=86e0154132bb6973fbfbd611efdf9b15, UNASSIGN in 1.8690 sec 2024-11-10T16:16:12,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-10T16:16:12,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-10T16:16:12,003 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731255372003"}]},"ts":"1731255372003"} 2024-11-10T16:16:12,004 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T16:16:12,004 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T16:16:12,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8790 sec 2024-11-10T16:16:12,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-10T16:16:12,273 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:12,274 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T16:16:12,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,275 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T16:16:12,276 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,278 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C, FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits] 2024-11-10T16:16:12,282 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1594778186c24f44b26f16c0919e500f to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/1594778186c24f44b26f16c0919e500f 2024-11-10T16:16:12,283 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/8b9e9e1a29b249f6af31724fd873f5c4 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/8b9e9e1a29b249f6af31724fd873f5c4 2024-11-10T16:16:12,283 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c3f6e427d2ac417d8d26d94ebec2110a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/A/c3f6e427d2ac417d8d26d94ebec2110a 2024-11-10T16:16:12,285 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5e662befb7784b0fbee7b9c09bfff86e to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/5e662befb7784b0fbee7b9c09bfff86e 2024-11-10T16:16:12,286 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/6bd4dea8eb7f46fb9aac304ee97b5d35 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/6bd4dea8eb7f46fb9aac304ee97b5d35 2024-11-10T16:16:12,287 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/c7cf5d7f8c4740ff8be7ad214e8c77da to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/B/c7cf5d7f8c4740ff8be7ad214e8c77da 2024-11-10T16:16:12,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/2f48022f03b8475e87630d56eb58d6f2 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/2f48022f03b8475e87630d56eb58d6f2 2024-11-10T16:16:12,289 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/5ac78c0159c04654aaf3ffa4da99b60a to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/5ac78c0159c04654aaf3ffa4da99b60a 2024-11-10T16:16:12,290 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/c344750c900f47cea12b96623027a6dd to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/C/c344750c900f47cea12b96623027a6dd 2024-11-10T16:16:12,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits/353.seqid to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15/recovered.edits/353.seqid 2024-11-10T16:16:12,292 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/default/TestAcidGuarantees/86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,292 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T16:16:12,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:16:12,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T16:16:12,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018e2e77ca931478495e53eef6719538e_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018e2e77ca931478495e53eef6719538e_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101dcb3b8bce2344789cbc29a0ddfeb7e7_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101dcb3b8bce2344789cbc29a0ddfeb7e7_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,297 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ac14be6096e42c9857f9f518e049642_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411103ac14be6096e42c9857f9f518e049642_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,298 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043f90d939a634b4e81e648945302c67a_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111043f90d939a634b4e81e648945302c67a_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044ce297b764144fdaa6f844adbd06aff_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044ce297b764144fdaa6f844adbd06aff_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,300 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fc9eab9ffa41078c0a8ea91e75f5ba_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111044fc9eab9ffa41078c0a8ea91e75f5ba_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048493e4c6b4b4c2f8eee2ea02ac919e8_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111048493e4c6b4b4c2f8eee2ea02ac919e8_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053b1ec5b2b004cf7af813aedc529e254_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053b1ec5b2b004cf7af813aedc529e254_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,302 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053dcd3c0557b40db87b498f1e7ebf9e0_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111053dcd3c0557b40db87b498f1e7ebf9e0_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,303 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105e4d52d9ada84a1eb89c66b00883dab5_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105e4d52d9ada84a1eb89c66b00883dab5_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,304 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076f75ec5680a462facbfbf3c307f717c_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111076f75ec5680a462facbfbf3c307f717c_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,305 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e6befbadb464feb830e09fb78a04d34_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e6befbadb464feb830e09fb78a04d34_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,305 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e9bdf513cdf4c38afa933f53d52e9bc_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107e9bdf513cdf4c38afa933f53d52e9bc_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,306 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a965a859bf04fc4bd05e5bfaa268788_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411108a965a859bf04fc4bd05e5bfaa268788_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,307 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd32b7f706aa49e185269919eab2b5d8_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110cd32b7f706aa49e185269919eab2b5d8_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd5053747ae04ce9ba66a75584f8a6dc_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd5053747ae04ce9ba66a75584f8a6dc_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd96f1072f434f58a8921d89be74d660_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dd96f1072f434f58a8921d89be74d660_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed9b606f4d2b420f9d4b49228e3148a2_86e0154132bb6973fbfbd611efdf9b15 to hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ed9b606f4d2b420f9d4b49228e3148a2_86e0154132bb6973fbfbd611efdf9b15 2024-11-10T16:16:12,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T16:16:12,312 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,313 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T16:16:12,315 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T16:16:12,316 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,316 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T16:16:12,316 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731255372316"}]},"ts":"9223372036854775807"} 2024-11-10T16:16:12,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T16:16:12,318 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 86e0154132bb6973fbfbd611efdf9b15, NAME => 'TestAcidGuarantees,,1731255342544.86e0154132bb6973fbfbd611efdf9b15.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T16:16:12,318 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T16:16:12,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731255372318"}]},"ts":"9223372036854775807"} 2024-11-10T16:16:12,319 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T16:16:12,320 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T16:16:12,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-10T16:16:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41631 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-10T16:16:12,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T16:16:12,383 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T16:16:12,393 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250 (was 250), OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 313), ProcessCount=11 (was 11), AvailableMemoryMB=6971 (was 7018) 2024-11-10T16:16:12,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T16:16:12,393 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:16:12,393 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:16:12,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:12,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:12,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T16:16:12,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T16:16:12,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1388394555, stopped=false 2024-11-10T16:16:12,394 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8e3de9160f9f,41631,1731255199363 2024-11-10T16:16:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T16:16:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T16:16:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:16:12,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:16:12,396 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T16:16:12,396 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T16:16:12,397 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T16:16:12,397 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:16:12,397 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T16:16:12,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:12,397 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8e3de9160f9f,44105,1731255200149' ***** 2024-11-10T16:16:12,397 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(959): stopping server 8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8e3de9160f9f:44105. 2024-11-10T16:16:12,398 DEBUG [RS:0;8e3de9160f9f:44105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T16:16:12,398 DEBUG [RS:0;8e3de9160f9f:44105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T16:16:12,398 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T16:16:12,399 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T16:16:12,399 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T16:16:12,399 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T16:16:12,399 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T16:16:12,399 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T16:16:12,399 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T16:16:12,399 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T16:16:12,399 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T16:16:12,399 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:12,400 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T16:16:12,423 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/info/55a3f3d60580460b9ee99e0a97246ece is 95, key is TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b./info:/1731255227954/DeleteFamily/seqid=0 2024-11-10T16:16:12,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742426_1602 (size=6514) 2024-11-10T16:16:12,495 INFO [regionserver/8e3de9160f9f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T16:16:12,600 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:12,800 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:12,826 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/info/55a3f3d60580460b9ee99e0a97246ece 2024-11-10T16:16:12,844 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/ns/cb716c7a6aaf46ccbc10ecb9c7c71ffb is 93, key is TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b./ns:/1731255227907/DeleteFamily/seqid=0 2024-11-10T16:16:12,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742427_1603 (size=5906) 2024-11-10T16:16:13,000 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:13,200 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:13,245 INFO [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T16:16:13,245 INFO [regionserver/8e3de9160f9f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T16:16:13,248 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/ns/cb716c7a6aaf46ccbc10ecb9c7c71ffb 2024-11-10T16:16:13,266 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/rep_barrier/a66ddb95acb14dcd8e0e5bb0c4da0226 is 102, key is TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b./rep_barrier:/1731255227907/DeleteFamily/seqid=0 2024-11-10T16:16:13,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742428_1604 (size=6025) 2024-11-10T16:16:13,401 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:13,601 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:13,670 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/rep_barrier/a66ddb95acb14dcd8e0e5bb0c4da0226 2024-11-10T16:16:13,688 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/table/7de862fd2acb4eeaa66567cfb8d5e0db is 96, key is TestAcidGuarantees,,1731255203775.c8fb0b94c7d60b2a8bd105a3337e427b./table:/1731255227907/DeleteFamily/seqid=0 2024-11-10T16:16:13,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742429_1605 (size=5936) 2024-11-10T16:16:13,801 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:14,001 DEBUG [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T16:16:14,091 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/table/7de862fd2acb4eeaa66567cfb8d5e0db 2024-11-10T16:16:14,095 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/info/55a3f3d60580460b9ee99e0a97246ece as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/info/55a3f3d60580460b9ee99e0a97246ece 2024-11-10T16:16:14,097 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/info/55a3f3d60580460b9ee99e0a97246ece, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T16:16:14,098 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/ns/cb716c7a6aaf46ccbc10ecb9c7c71ffb as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/ns/cb716c7a6aaf46ccbc10ecb9c7c71ffb 2024-11-10T16:16:14,100 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/ns/cb716c7a6aaf46ccbc10ecb9c7c71ffb, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T16:16:14,101 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/rep_barrier/a66ddb95acb14dcd8e0e5bb0c4da0226 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/rep_barrier/a66ddb95acb14dcd8e0e5bb0c4da0226 2024-11-10T16:16:14,103 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/rep_barrier/a66ddb95acb14dcd8e0e5bb0c4da0226, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T16:16:14,104 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/.tmp/table/7de862fd2acb4eeaa66567cfb8d5e0db as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/table/7de862fd2acb4eeaa66567cfb8d5e0db 2024-11-10T16:16:14,106 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/table/7de862fd2acb4eeaa66567cfb8d5e0db, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T16:16:14,107 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false 2024-11-10T16:16:14,110 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T16:16:14,111 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T16:16:14,111 INFO [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T16:16:14,111 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731255372399Running coprocessor pre-close hooks at 1731255372399Disabling compacts and flushes for region at 1731255372399Disabling writes for close at 1731255372399Obtaining lock to block concurrent updates at 1731255372400 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731255372400Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731255372400Flushing stores of hbase:meta,,1.1588230740 at 1731255372401 (+1 ms)Flushing 1588230740/info: creating writer at 1731255372401Flushing 1588230740/info: appending metadata at 1731255372422 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731255372422Flushing 1588230740/ns: creating writer at 1731255372830 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731255372844 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731255372844Flushing 1588230740/rep_barrier: creating writer at 1731255373251 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731255373266 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731255373266Flushing 1588230740/table: creating writer at 1731255373673 (+407 ms)Flushing 1588230740/table: appending metadata at 1731255373687 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731255373687Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bf97543: reopening flushed file at 1731255374094 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35bbec2e: reopening flushed file at 1731255374097 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a63777e: reopening flushed file at 1731255374100 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36290a29: reopening flushed file at 1731255374103 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1707ms, sequenceid=90, compaction requested=false at 1731255374107 (+4 ms)Writing region close event to WAL at 1731255374108 (+1 ms)Running coprocessor post-close hooks at 1731255374111 (+3 ms)Closed at 1731255374111 2024-11-10T16:16:14,111 DEBUG [RS_CLOSE_META-regionserver/8e3de9160f9f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T16:16:14,201 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(976): stopping server 8e3de9160f9f,44105,1731255200149; all regions closed. 2024-11-10T16:16:14,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741834_1010 (size=25344) 2024-11-10T16:16:14,208 DEBUG [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/oldWALs 2024-11-10T16:16:14,208 INFO [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8e3de9160f9f%2C44105%2C1731255200149.meta:.meta(num 1731255203376) 2024-11-10T16:16:14,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741832_1008 (size=14140958) 2024-11-10T16:16:14,211 DEBUG [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/oldWALs 2024-11-10T16:16:14,211 INFO [RS:0;8e3de9160f9f:44105 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8e3de9160f9f%2C44105%2C1731255200149:(num 1731255202404) 2024-11-10T16:16:14,211 DEBUG [RS:0;8e3de9160f9f:44105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T16:16:14,211 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T16:16:14,211 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T16:16:14,211 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.ChoreService(370): Chore service for: regionserver/8e3de9160f9f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T16:16:14,211 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T16:16:14,212 INFO [regionserver/8e3de9160f9f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T16:16:14,212 INFO [RS:0;8e3de9160f9f:44105 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44105 2024-11-10T16:16:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8e3de9160f9f,44105,1731255200149 2024-11-10T16:16:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T16:16:14,216 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T16:16:14,220 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8e3de9160f9f,44105,1731255200149] 2024-11-10T16:16:14,221 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8e3de9160f9f,44105,1731255200149 already deleted, retry=false 2024-11-10T16:16:14,222 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8e3de9160f9f,44105,1731255200149 expired; onlineServers=0 2024-11-10T16:16:14,222 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8e3de9160f9f,41631,1731255199363' ***** 2024-11-10T16:16:14,222 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T16:16:14,222 INFO [M:0;8e3de9160f9f:41631 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T16:16:14,222 INFO [M:0;8e3de9160f9f:41631 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T16:16:14,222 DEBUG [M:0;8e3de9160f9f:41631 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T16:16:14,222 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T16:16:14,222 DEBUG [M:0;8e3de9160f9f:41631 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T16:16:14,222 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731255202116 {}] cleaner.HFileCleaner(306): Exit Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.large.0-1731255202116,5,FailOnTimeoutGroup] 2024-11-10T16:16:14,222 DEBUG [master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731255202117 {}] cleaner.HFileCleaner(306): Exit Thread[master/8e3de9160f9f:0:becomeActiveMaster-HFileCleaner.small.0-1731255202117,5,FailOnTimeoutGroup] 2024-11-10T16:16:14,222 INFO [M:0;8e3de9160f9f:41631 {}] hbase.ChoreService(370): Chore service for: master/8e3de9160f9f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T16:16:14,222 INFO [M:0;8e3de9160f9f:41631 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T16:16:14,223 DEBUG [M:0;8e3de9160f9f:41631 {}] master.HMaster(1795): Stopping service threads 2024-11-10T16:16:14,223 INFO [M:0;8e3de9160f9f:41631 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T16:16:14,223 INFO [M:0;8e3de9160f9f:41631 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T16:16:14,223 ERROR [M:0;8e3de9160f9f:41631 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:40533 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40533,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T16:16:14,223 INFO [M:0;8e3de9160f9f:41631 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T16:16:14,223 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T16:16:14,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T16:16:14,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T16:16:14,225 DEBUG [M:0;8e3de9160f9f:41631 {}] zookeeper.ZKUtil(347): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T16:16:14,225 WARN [M:0;8e3de9160f9f:41631 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T16:16:14,225 INFO [M:0;8e3de9160f9f:41631 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/.lastflushedseqids 2024-11-10T16:16:14,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742430_1606 (size=72) 2024-11-10T16:16:14,234 INFO [M:0;8e3de9160f9f:41631 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T16:16:14,235 INFO [M:0;8e3de9160f9f:41631 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T16:16:14,235 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T16:16:14,235 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:16:14,235 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:16:14,235 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T16:16:14,235 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:16:14,235 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=769.88 KB heapSize=942.42 KB 2024-11-10T16:16:14,250 DEBUG [M:0;8e3de9160f9f:41631 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a1ee9fc11f44088b73b976363371de0 is 82, key is hbase:meta,,1/info:regioninfo/1731255203463/Put/seqid=0 2024-11-10T16:16:14,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742431_1607 (size=5672) 2024-11-10T16:16:14,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T16:16:14,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44105-0x10102a7cd7a0001, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T16:16:14,321 INFO [RS:0;8e3de9160f9f:44105 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T16:16:14,321 INFO [RS:0;8e3de9160f9f:44105 {}] regionserver.HRegionServer(1031): Exiting; stopping=8e3de9160f9f,44105,1731255200149; zookeeper connection closed. 2024-11-10T16:16:14,321 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ba646bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ba646bc 2024-11-10T16:16:14,322 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T16:16:14,653 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a1ee9fc11f44088b73b976363371de0 2024-11-10T16:16:14,674 DEBUG [M:0;8e3de9160f9f:41631 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/154b164972774df787a74061d125c139 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731255345611/Put/seqid=0 2024-11-10T16:16:14,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742432_1608 (size=45005) 2024-11-10T16:16:15,078 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=769.32 KB at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/154b164972774df787a74061d125c139 2024-11-10T16:16:15,082 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 154b164972774df787a74061d125c139 2024-11-10T16:16:15,096 DEBUG [M:0;8e3de9160f9f:41631 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a342fec379bf4753ba2ff0d2dee49a4a is 69, key is 8e3de9160f9f,44105,1731255200149/rs:state/1731255202171/Put/seqid=0 2024-11-10T16:16:15,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073742433_1609 (size=5156) 2024-11-10T16:16:15,500 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2131 (bloomFilter=true), to=hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a342fec379bf4753ba2ff0d2dee49a4a 2024-11-10T16:16:15,503 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a1ee9fc11f44088b73b976363371de0 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a1ee9fc11f44088b73b976363371de0 2024-11-10T16:16:15,505 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a1ee9fc11f44088b73b976363371de0, entries=8, sequenceid=2131, filesize=5.5 K 2024-11-10T16:16:15,506 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/154b164972774df787a74061d125c139 as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/154b164972774df787a74061d125c139 2024-11-10T16:16:15,509 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 154b164972774df787a74061d125c139 2024-11-10T16:16:15,509 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/154b164972774df787a74061d125c139, entries=168, sequenceid=2131, filesize=44.0 K 2024-11-10T16:16:15,509 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a342fec379bf4753ba2ff0d2dee49a4a as hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a342fec379bf4753ba2ff0d2dee49a4a 2024-11-10T16:16:15,512 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40533/user/jenkins/test-data/869b902b-b92e-2b65-54e7-5df2fbec8255/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a342fec379bf4753ba2ff0d2dee49a4a, entries=1, sequenceid=2131, filesize=5.0 K 2024-11-10T16:16:15,513 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(3140): Finished flush of dataSize ~769.88 KB/788352, heapSize ~942.13 KB/964736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2131, compaction requested=false 2024-11-10T16:16:15,514 INFO [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T16:16:15,514 DEBUG [M:0;8e3de9160f9f:41631 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731255374235Disabling compacts and flushes for region at 1731255374235Disabling writes for close at 1731255374235Obtaining lock to block concurrent updates at 1731255374235Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731255374235Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=788352, getHeapSize=964976, getOffHeapSize=0, getCellsCount=2324 at 1731255374235Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731255374236 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731255374236Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731255374249 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731255374249Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731255374656 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731255374674 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731255374674Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731255375082 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731255375096 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731255375096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e4db343: reopening flushed file at 1731255375502 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4013c018: reopening flushed file at 1731255375505 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@329e16bf: reopening flushed file at 1731255375509 (+4 ms)Finished flush of dataSize ~769.88 KB/788352, heapSize ~942.13 KB/964736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1278ms, sequenceid=2131, compaction requested=false at 1731255375513 (+4 ms)Writing region close event to WAL at 1731255375514 (+1 ms)Closed at 1731255375514 2024-11-10T16:16:15,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37515 is added to blk_1073741830_1006 (size=928058) 2024-11-10T16:16:15,517 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T16:16:15,517 INFO [M:0;8e3de9160f9f:41631 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T16:16:15,517 INFO [M:0;8e3de9160f9f:41631 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41631 2024-11-10T16:16:15,518 INFO [M:0;8e3de9160f9f:41631 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T16:16:15,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T16:16:15,620 INFO [M:0;8e3de9160f9f:41631 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T16:16:15,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41631-0x10102a7cd7a0000, quorum=127.0.0.1:50949, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T16:16:15,625 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T16:16:15,627 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T16:16:15,627 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T16:16:15,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T16:16:15,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.log.dir/,STOPPED} 2024-11-10T16:16:15,630 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T16:16:15,630 WARN [BP-1564034578-172.17.0.2-1731255195956 heartbeating to localhost/127.0.0.1:40533 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T16:16:15,630 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T16:16:15,630 WARN [BP-1564034578-172.17.0.2-1731255195956 heartbeating to localhost/127.0.0.1:40533 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1564034578-172.17.0.2-1731255195956 (Datanode Uuid 4bdfe523-a046-41fd-9c51-0ee6e262a8d8) service to localhost/127.0.0.1:40533 2024-11-10T16:16:15,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data1/current/BP-1564034578-172.17.0.2-1731255195956 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T16:16:15,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/cluster_02ce6b7e-fa17-b73c-ddd0-883e2070068c/data/data2/current/BP-1564034578-172.17.0.2-1731255195956 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T16:16:15,633 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T16:16:15,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T16:16:15,640 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T16:16:15,640 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T16:16:15,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T16:16:15,641 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/df2509ba-b44e-50ff-a1c2-3261e0d5b487/hadoop.log.dir/,STOPPED} 2024-11-10T16:16:15,655 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T16:16:15,765 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down